From a93a09e0f727f34390bf9946a31683eceac50088 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Tue, 29 Apr 2025 09:57:42 +0900 Subject: [PATCH 1/6] feat: clean up message_files table first before proceeding to find orphaned files (#19035) --- api/commands.py | 71 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/api/commands.py b/api/commands.py index 3881439ddf..99a3211baf 100644 --- a/api/commands.py +++ b/api/commands.py @@ -818,8 +818,9 @@ def clear_free_plan_tenant_expired_logs(days: int, batch: int, tenant_ids: list[ click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green")) +@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.") @click.command("clear-orphaned-file-records", help="Clear orphaned file records.") -def clear_orphaned_file_records(): +def clear_orphaned_file_records(force: bool): """ Clear orphaned file records in the database. """ @@ -845,7 +846,15 @@ def clear_orphaned_file_records(): # notify user and ask for confirmation click.echo( - click.style("This command will find and delete orphaned file records in the following tables:", fg="yellow") + click.style( + "This command will first find and delete orphaned file records from the message_files table,", fg="yellow" + ) + ) + click.echo( + click.style( + "and then it will find and delete orphaned file records in the following tables:", + fg="yellow", + ) ) for files_table in files_tables: click.echo(click.style(f"- {files_table['table']}", fg="yellow")) @@ -878,11 +887,55 @@ def clear_orphaned_file_records(): fg="yellow", ) ) - click.confirm("Do you want to proceed?", abort=True) + if not force: + click.confirm("Do you want to proceed?", abort=True) # start the cleanup process click.echo(click.style("Starting orphaned file records cleanup.", fg="white")) + # clean up the orphaned records in the message_files table where message_id doesn't exist in messages table + try: + click.echo( + click.style("- Listing message_files records where message_id doesn't exist in messages table", fg="white") + ) + query = ( + "SELECT mf.id, mf.message_id " + "FROM message_files mf LEFT JOIN messages m ON mf.message_id = m.id " + "WHERE m.id IS NULL" + ) + orphaned_message_files = [] + with db.engine.begin() as conn: + rs = conn.execute(db.text(query)) + for i in rs: + orphaned_message_files.append({"id": str(i[0]), "message_id": str(i[1])}) + + if orphaned_message_files: + click.echo(click.style(f"Found {len(orphaned_message_files)} orphaned message_files records:", fg="white")) + for record in orphaned_message_files: + click.echo(click.style(f" - id: {record['id']}, message_id: {record['message_id']}", fg="black")) + + if not force: + click.confirm( + ( + f"Do you want to proceed " + f"to delete all {len(orphaned_message_files)} orphaned message_files records?" + ), + abort=True, + ) + + click.echo(click.style("- Deleting orphaned message_files records", fg="white")) + query = "DELETE FROM message_files WHERE id IN :ids" + with db.engine.begin() as conn: + conn.execute(db.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])}) + click.echo( + click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green") + ) + else: + click.echo(click.style("No orphaned message_files records found. There is nothing to delete.", fg="green")) + except Exception as e: + click.echo(click.style(f"Error deleting orphaned message_files records: {str(e)}", fg="red")) + + # clean up the orphaned records in the rest of the *_files tables try: # fetch file id and keys from each table all_files_in_tables = [] @@ -964,7 +1017,8 @@ def clear_orphaned_file_records(): click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white")) for file in orphaned_files: click.echo(click.style(f"- orphaned file id: {file}", fg="black")) - click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True) + if not force: + click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True) # delete orphaned records for each file try: @@ -979,8 +1033,9 @@ def clear_orphaned_file_records(): click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green")) +@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.") @click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.") -def remove_orphaned_files_on_storage(): +def remove_orphaned_files_on_storage(force: bool): """ Remove orphaned files on the storage. """ @@ -1028,7 +1083,8 @@ def remove_orphaned_files_on_storage(): fg="yellow", ) ) - click.confirm("Do you want to proceed?", abort=True) + if not force: + click.confirm("Do you want to proceed?", abort=True) # start the cleanup process click.echo(click.style("Starting orphaned files cleanup.", fg="white")) @@ -1069,7 +1125,8 @@ def remove_orphaned_files_on_storage(): click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white")) for file in orphaned_files: click.echo(click.style(f"- orphaned file: {file}", fg="black")) - click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True) + if not force: + click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True) # delete orphaned files removed_files = 0 From 36521e42759d519a824d6d851c32d470a4f39bca Mon Sep 17 00:00:00 2001 From: Kalo Chin <91766386+fdb02983rhy@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:57:58 +0900 Subject: [PATCH 2/6] fixes: fix entrypoint script with missing environment variables (#19039) --- web/docker/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index adb1d7d3af..59e9c472b3 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -32,4 +32,7 @@ export NEXT_PUBLIC_MAX_TOOLS_NUM=${MAX_TOOLS_NUM} export NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER=${ENABLE_WEBSITE_JINAREADER:-true} export NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL=${ENABLE_WEBSITE_FIRECRAWL:-true} export NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL=${ENABLE_WEBSITE_WATERCRAWL:-true} +export NEXT_PUBLIC_LOOP_NODE_MAX_COUNT=${LOOP_NODE_MAX_COUNT} +export NEXT_PUBLIC_MAX_PARALLEL_LIMIT=${MAX_PARALLEL_LIMIT} +export NEXT_PUBLIC_MAX_ITERATIONS_NUM=${MAX_ITERATIONS_NUM} pm2 start /app/web/server.js --name dify-web --cwd /app/web -i ${PM2_INSTANCES} --no-daemon From f0ca828544b10c2429f28410c5eed26aea29568f Mon Sep 17 00:00:00 2001 From: Panpan Date: Tue, 29 Apr 2025 08:58:10 +0800 Subject: [PATCH 3/6] fix: fix embedded chatbot styles on a relatively wide screen (#19030) --- web/public/embed.js | 2 +- web/public/embed.min.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/public/embed.js b/web/public/embed.js index 7060a97053..0d6d72f909 100644 --- a/web/public/embed.js +++ b/web/public/embed.js @@ -57,7 +57,7 @@ left: unset; min-width: 24rem; width: 48%; - max-width: calc(100vw - 2rem); + max-width: 40rem; /* Match mobile breakpoint*/ min-height: 43.75rem; height: 88%; max-height: calc(100vh - 6rem); diff --git a/web/public/embed.min.js b/web/public/embed.min.js index 8bcdfcbbec..0ba42eb508 100644 --- a/web/public/embed.min.js +++ b/web/public/embed.min.js @@ -39,4 +39,4 @@ - `,n.appendChild(e),document.body.appendChild(n),n.addEventListener("click",t),n.addEventListener("touchend",t),y.draggable){var a=n;var l=y.dragAxis||"both";let s,d,t,r;function o(e){u=!1,r=("touchstart"===e.type?(s=e.touches[0].clientX-a.offsetLeft,d=e.touches[0].clientY-a.offsetTop,t=e.touches[0].clientX,e.touches[0]):(s=e.clientX-a.offsetLeft,d=e.clientY-a.offsetTop,t=e.clientX,e)).clientY,document.addEventListener("mousemove",i),document.addEventListener("touchmove",i,{passive:!1}),document.addEventListener("mouseup",c),document.addEventListener("touchend",c),e.preventDefault()}function i(n){var o="touchmove"===n.type?n.touches[0]:n,i=o.clientX-t,o=o.clientY-r;if(u=8{u=!1},0),a.style.transition="",a.style.cursor="pointer",document.removeEventListener("mousemove",i),document.removeEventListener("touchmove",i),document.removeEventListener("mouseup",c),document.removeEventListener("touchend",c)}a.addEventListener("mousedown",o),a.addEventListener("touchstart",o)}}n.style.display="none",document.body.appendChild(n),2048{var t,n;e.origin===o&&(t=document.getElementById(m))&&e.source===t.contentWindow&&("dify-chatbot-iframe-ready"===e.data.type&&t.contentWindow?.postMessage({type:"dify-chatbot-config",payload:{isToggledByButton:!0,isDraggable:!!y.draggable}},o),"dify-chatbot-expand-change"===e.data.type)&&(a=!a,n=document.getElementById(m))&&(a?n.style.cssText="\n position: absolute;\n display: flex;\n flex-direction: column;\n justify-content: space-between;\n top: unset;\n right: var(--dify-chatbot-bubble-button-right, 1rem); /* Align with dify-chatbot-bubble-button. */\n bottom: var(--dify-chatbot-bubble-button-bottom, 1rem); /* Align with dify-chatbot-bubble-button. */\n left: unset;\n min-width: 24rem;\n width: 48%;\n max-width: calc(100vw - 2rem);\n min-height: 43.75rem;\n height: 88%;\n max-height: calc(100vh - 6rem);\n border: none;\n z-index: 2147483640;\n overflow: hidden;\n user-select: none;\n transition-property: width, height;\n transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);\n transition-duration: 150ms;\n ":n.style.cssText=l,d())}),document.getElementById(h)||r()}else console.error(t+" is empty or token is not provided")}function p(e="open"){"open"===e?(document.getElementById("openIcon").style.display="block",document.getElementById("closeIcon").style.display="none"):(document.getElementById("openIcon").style.display="none",document.getElementById("closeIcon").style.display="block")}function b(e){"Escape"===e.key&&(e=document.getElementById(m))&&"none"!==e.style.display&&(e.style.display="none",p("open"))}h,h,document.addEventListener("keydown",b),y?.dynamicScript?e():document.body.onload=e})(); \ No newline at end of file + `,n.appendChild(e),document.body.appendChild(n),n.addEventListener("click",t),n.addEventListener("touchend",t),y.draggable){var a=n;var l=y.dragAxis||"both";let s,d,t,r;function o(e){u=!1,r=("touchstart"===e.type?(s=e.touches[0].clientX-a.offsetLeft,d=e.touches[0].clientY-a.offsetTop,t=e.touches[0].clientX,e.touches[0]):(s=e.clientX-a.offsetLeft,d=e.clientY-a.offsetTop,t=e.clientX,e)).clientY,document.addEventListener("mousemove",i),document.addEventListener("touchmove",i,{passive:!1}),document.addEventListener("mouseup",c),document.addEventListener("touchend",c),e.preventDefault()}function i(n){var o="touchmove"===n.type?n.touches[0]:n,i=o.clientX-t,o=o.clientY-r;if(u=8{u=!1},0),a.style.transition="",a.style.cursor="pointer",document.removeEventListener("mousemove",i),document.removeEventListener("touchmove",i),document.removeEventListener("mouseup",c),document.removeEventListener("touchend",c)}a.addEventListener("mousedown",o),a.addEventListener("touchstart",o)}}n.style.display="none",document.body.appendChild(n),2048{var t,n;e.origin===o&&(t=document.getElementById(m))&&e.source===t.contentWindow&&("dify-chatbot-iframe-ready"===e.data.type&&t.contentWindow?.postMessage({type:"dify-chatbot-config",payload:{isToggledByButton:!0,isDraggable:!!y.draggable}},o),"dify-chatbot-expand-change"===e.data.type)&&(a=!a,n=document.getElementById(m))&&(a?n.style.cssText="\n position: absolute;\n display: flex;\n flex-direction: column;\n justify-content: space-between;\n top: unset;\n right: var(--dify-chatbot-bubble-button-right, 1rem); /* Align with dify-chatbot-bubble-button. */\n bottom: var(--dify-chatbot-bubble-button-bottom, 1rem); /* Align with dify-chatbot-bubble-button. */\n left: unset;\n min-width: 24rem;\n width: 48%;\n max-width: 40rem; /* Match mobile breakpoint*/\n min-height: 43.75rem;\n height: 88%;\n max-height: calc(100vh - 6rem);\n border: none;\n z-index: 2147483640;\n overflow: hidden;\n user-select: none;\n transition-property: width, height;\n transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);\n transition-duration: 150ms;\n ":n.style.cssText=l,d())}),document.getElementById(h)||r()}else console.error(t+" is empty or token is not provided")}function p(e="open"){"open"===e?(document.getElementById("openIcon").style.display="block",document.getElementById("closeIcon").style.display="none"):(document.getElementById("openIcon").style.display="none",document.getElementById("closeIcon").style.display="block")}function b(e){"Escape"===e.key&&(e=document.getElementById(m))&&"none"!==e.style.display&&(e.style.display="none",p("open"))}h,h,document.addEventListener("keydown",b),y?.dynamicScript?e():document.body.onload=e})(); \ No newline at end of file From bf46b894e98707dfd6825dc93f25e2452c9f2852 Mon Sep 17 00:00:00 2001 From: Kalo Chin <91766386+fdb02983rhy@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:05:04 +0900 Subject: [PATCH 4/6] chore: Improve FILES_URL Configuration Comments (#19040) --- docker/.env.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 83d975cec5..1adb07ca64 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -39,6 +39,12 @@ APP_WEB_URL= # File preview or download Url prefix. # used to display File preview or download Url to the front-end or as Multi-model inputs; # Url is signed and has expiration time. +# Setting FILES_URL is required for file processing plugins. +# - For https://example.com, use FILES_URL=https://example.com +# - For http://example.com, use FILES_URL=http://example.com +# Recommendation: use a dedicated domain (e.g., https://upload.example.com). +# Alternatively, use http://:5001 or http://api:5001, +# ensuring port 5001 is externally accessible (see docker-compose.yaml). FILES_URL= # ------------------------------ From e36b1a7016f64bebf8811a8e1cb47c6032f49374 Mon Sep 17 00:00:00 2001 From: Novice <857526207@qq.com> Date: Tue, 29 Apr 2025 09:51:42 +0800 Subject: [PATCH 5/6] test(graph-engine-test): modify the assert condition (#19041) --- .../core/workflow/graph_engine/test_graph_engine.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_graph_engine.py b/api/tests/unit_tests/core/workflow/graph_engine/test_graph_engine.py index 2a29ad3e41..f3dbd1836b 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_graph_engine.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_graph_engine.py @@ -864,10 +864,11 @@ def test_condition_parallel_correct_output(mock_close, mock_remove, app): with patch.object(CodeNode, "_run", new=code_generator): generator = graph_engine.run() stream_content = "" - res_content = "VAT:\ndify 123" + wrong_content = ["Stamp Duty", "other"] for item in generator: if isinstance(item, NodeRunStreamChunkEvent): stream_content += f"{item.chunk_content}\n" if isinstance(item, GraphRunSucceededEvent): - assert item.outputs == {"answer": res_content} - assert stream_content == res_content + "\n" + assert item.outputs is not None + answer = item.outputs["answer"] + assert all(rc not in answer for rc in wrong_content) From 94cc0b7a121f5e6dd11b4c17424c1d677c8fc7c2 Mon Sep 17 00:00:00 2001 From: Kevin9703 <51311316+Kevin9703@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:31:08 +0800 Subject: [PATCH 6/6] fix(workflow_cycle_manage): failed nodes were not updated in workflow_node_executions (#18994) --- api/core/app/task_pipeline/workflow_cycle_manage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/core/app/task_pipeline/workflow_cycle_manage.py b/api/core/app/task_pipeline/workflow_cycle_manage.py index d720eff73d..09e2ee74e6 100644 --- a/api/core/app/task_pipeline/workflow_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_cycle_manage.py @@ -381,6 +381,8 @@ class WorkflowCycleManage: workflow_node_execution.elapsed_time = elapsed_time workflow_node_execution.execution_metadata = execution_metadata + self._workflow_node_execution_repository.update(workflow_node_execution) + return workflow_node_execution def _handle_workflow_node_execution_retried(