From b62eb61400ae77a258329093b42a4aa5e8aaeeef Mon Sep 17 00:00:00 2001 From: Wesley Date: Sun, 27 Apr 2025 04:04:56 +0100 Subject: [PATCH 01/10] fix depth param issue for WaterCrawl (#18839) --- api/core/rag/extractor/watercrawl/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/rag/extractor/watercrawl/provider.py b/api/core/rag/extractor/watercrawl/provider.py index b8003b386b..21fbb2100f 100644 --- a/api/core/rag/extractor/watercrawl/provider.py +++ b/api/core/rag/extractor/watercrawl/provider.py @@ -20,7 +20,7 @@ class WaterCrawlProvider: } if options.get("crawl_sub_pages", True): spider_options["page_limit"] = options.get("limit", 1) - spider_options["max_depth"] = options.get("depth", 1) + spider_options["max_depth"] = options.get("max_depth", 1) spider_options["include_paths"] = options.get("includes", "").split(",") if options.get("includes") else [] spider_options["exclude_paths"] = options.get("excludes", "").split(",") if options.get("excludes") else [] From 993ef87dcace81f02ee5aaac980e7e3fa3246962 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Sun, 27 Apr 2025 12:11:04 +0900 Subject: [PATCH 02/10] feat: add administrative commands to free up storage space by removing unused files (#18835) --- api/commands.py | 254 ++++++++++++++++++++++ api/extensions/ext_commands.py | 4 + api/extensions/ext_storage.py | 3 + api/extensions/storage/base_storage.py | 8 + api/extensions/storage/opendal_storage.py | 17 ++ 5 files changed, 286 insertions(+) diff --git a/api/commands.py b/api/commands.py index c5394c6f87..f1ff8e1401 100644 --- a/api/commands.py +++ b/api/commands.py @@ -17,6 +17,7 @@ from core.rag.models.document import Document from events.app_event import app_was_created from extensions.ext_database import db from extensions.ext_redis import redis_client +from extensions.ext_storage import storage from libs.helper import email as email_validate from libs.password import hash_password, password_pattern, valid_password from libs.rsa import generate_key_pair @@ -815,3 +816,256 @@ def clear_free_plan_tenant_expired_logs(days: int, batch: int, tenant_ids: list[ ClearFreePlanTenantExpiredLogs.process(days, batch, tenant_ids) click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green")) + + +@click.command("clear-orphaned-file-records", help="Clear orphaned file records.") +def clear_orphaned_file_records(): + """ + Clear orphaned file records in the database. + """ + + # define tables and columns to process + files_tables = [ + {"table": "upload_files", "id_column": "id", "key_column": "key"}, + {"table": "tool_files", "id_column": "id", "key_column": "file_key"}, + ] + ids_tables = [ + {"type": "uuid", "table": "message_files", "column": "upload_file_id"}, + {"type": "text", "table": "documents", "column": "data_source_info"}, + {"type": "text", "table": "document_segments", "column": "content"}, + {"type": "text", "table": "messages", "column": "answer"}, + {"type": "text", "table": "workflow_node_executions", "column": "inputs"}, + {"type": "text", "table": "workflow_node_executions", "column": "process_data"}, + {"type": "text", "table": "workflow_node_executions", "column": "outputs"}, + {"type": "text", "table": "conversations", "column": "introduction"}, + {"type": "text", "table": "conversations", "column": "system_instruction"}, + {"type": "json", "table": "messages", "column": "inputs"}, + {"type": "json", "table": "messages", "column": "message"}, + ] + + # notify user and ask for confirmation + click.echo( + click.style("This command will find and delete orphaned file records in the following tables:", fg="yellow") + ) + for files_table in files_tables: + click.echo(click.style(f"- {files_table['table']}", fg="yellow")) + click.echo( + click.style("The following tables and columns will be scanned to find orphaned file records:", fg="yellow") + ) + for ids_table in ids_tables: + click.echo(click.style(f"- {ids_table['table']} ({ids_table['column']})", fg="yellow")) + click.echo("") + + click.echo(click.style("!!! USE WITH CAUTION !!!", fg="red")) + click.echo( + click.style( + ( + "Since not all patterns have been fully tested, " + "please note that this command may delete unintended file records." + ), + fg="yellow", + ) + ) + click.echo( + click.style("This cannot be undone. Please make sure to back up your database before proceeding.", fg="yellow") + ) + click.confirm("Do you want to proceed?", abort=True) + + # start the cleanup process + click.echo(click.style("Starting orphaned file records cleanup.", fg="white")) + + try: + # fetch file id and keys from each table + all_files_in_tables = [] + for files_table in files_tables: + click.echo(click.style(f"- Listing file records in table {files_table['table']}", fg="white")) + query = f"SELECT {files_table['id_column']}, {files_table['key_column']} FROM {files_table['table']}" + with db.engine.begin() as conn: + rs = conn.execute(db.text(query)) + for i in rs: + all_files_in_tables.append({"table": files_table["table"], "id": str(i[0]), "key": i[1]}) + click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white")) + + # fetch referred table and columns + guid_regexp = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" + all_ids_in_tables = [] + for ids_table in ids_tables: + query = "" + if ids_table["type"] == "uuid": + click.echo( + click.style( + f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}", fg="white" + ) + ) + query = ( + f"SELECT {ids_table['column']} FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL" + ) + with db.engine.begin() as conn: + rs = conn.execute(db.text(query)) + for i in rs: + all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])}) + elif ids_table["type"] == "text": + click.echo( + click.style( + f"- Listing file-id-like strings in column {ids_table['column']} in table {ids_table['table']}", + fg="white", + ) + ) + query = ( + f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id " + f"FROM {ids_table['table']}" + ) + with db.engine.begin() as conn: + rs = conn.execute(db.text(query)) + for i in rs: + for j in i[0]: + all_ids_in_tables.append({"table": ids_table["table"], "id": j}) + elif ids_table["type"] == "json": + click.echo( + click.style( + ( + f"- Listing file-id-like JSON string in column {ids_table['column']} " + f"in table {ids_table['table']}" + ), + fg="white", + ) + ) + query = ( + f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id " + f"FROM {ids_table['table']}" + ) + with db.engine.begin() as conn: + rs = conn.execute(db.text(query)) + for i in rs: + for j in i[0]: + all_ids_in_tables.append({"table": ids_table["table"], "id": j}) + click.echo(click.style(f"Found {len(all_ids_in_tables)} file ids in tables.", fg="white")) + + except Exception as e: + click.echo(click.style(f"Error fetching keys: {str(e)}", fg="red")) + return + + # find orphaned files + all_files = [file["id"] for file in all_files_in_tables] + all_ids = [file["id"] for file in all_ids_in_tables] + orphaned_files = list(set(all_files) - set(all_ids)) + if not orphaned_files: + click.echo(click.style("No orphaned file records found. There is nothing to delete.", fg="green")) + return + click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white")) + for file in orphaned_files: + click.echo(click.style(f"- orphaned file id: {file}", fg="black")) + click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True) + + # delete orphaned records for each file + try: + for files_table in files_tables: + click.echo(click.style(f"- Deleting orphaned file records in table {files_table['table']}", fg="white")) + query = f"DELETE FROM {files_table['table']} WHERE {files_table['id_column']} IN :ids" + with db.engine.begin() as conn: + conn.execute(db.text(query), {"ids": tuple(orphaned_files)}) + except Exception as e: + click.echo(click.style(f"Error deleting orphaned file records: {str(e)}", fg="red")) + return + click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green")) + + +@click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.") +def remove_orphaned_files_on_storage(): + """ + Remove orphaned files on the storage. + """ + + # define tables and columns to process + files_tables = [ + {"table": "upload_files", "key_column": "key"}, + {"table": "tool_files", "key_column": "file_key"}, + ] + storage_paths = ["image_files", "tools", "upload_files"] + + # notify user and ask for confirmation + click.echo(click.style("This command will find and remove orphaned files on the storage,", fg="yellow")) + click.echo( + click.style("by comparing the files on the storage with the records in the following tables:", fg="yellow") + ) + for files_table in files_tables: + click.echo(click.style(f"- {files_table['table']}", fg="yellow")) + click.echo(click.style("The following paths on the storage will be scanned to find orphaned files:", fg="yellow")) + for storage_path in storage_paths: + click.echo(click.style(f"- {storage_path}", fg="yellow")) + click.echo("") + + click.echo(click.style("!!! USE WITH CAUTION !!!", fg="red")) + click.echo( + click.style( + "Currently, this command will work only for opendal based storage (STORAGE_TYPE=opendal).", fg="yellow" + ) + ) + click.echo( + click.style( + "Since not all patterns have been fully tested, please note that this command may delete unintended files.", + fg="yellow", + ) + ) + click.echo( + click.style("This cannot be undone. Please make sure to back up your database before proceeding.", fg="yellow") + ) + click.confirm("Do you want to proceed?", abort=True) + + # start the cleanup process + click.echo(click.style("Starting orphaned files cleanup.", fg="white")) + + # fetch file id and keys from each table + all_files_in_tables = [] + try: + for files_table in files_tables: + click.echo(click.style(f"- Listing files from table {files_table['table']}", fg="white")) + query = f"SELECT {files_table['key_column']} FROM {files_table['table']}" + with db.engine.begin() as conn: + rs = conn.execute(db.text(query)) + for i in rs: + all_files_in_tables.append(str(i[0])) + click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white")) + except Exception as e: + click.echo(click.style(f"Error fetching keys: {str(e)}", fg="red")) + + all_files_on_storage = [] + for storage_path in storage_paths: + try: + click.echo(click.style(f"- Scanning files on storage path {storage_path}", fg="white")) + files = storage.scan(path=storage_path, files=True, directories=False) + all_files_on_storage.extend(files) + except FileNotFoundError as e: + click.echo(click.style(f" -> Skipping path {storage_path} as it does not exist.", fg="yellow")) + continue + except Exception as e: + click.echo(click.style(f" -> Error scanning files on storage path {storage_path}: {str(e)}", fg="red")) + continue + click.echo(click.style(f"Found {len(all_files_on_storage)} files on storage.", fg="white")) + + # find orphaned files + orphaned_files = list(set(all_files_on_storage) - set(all_files_in_tables)) + if not orphaned_files: + click.echo(click.style("No orphaned files found. There is nothing to remove.", fg="green")) + return + click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white")) + for file in orphaned_files: + click.echo(click.style(f"- orphaned file: {file}", fg="black")) + click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True) + + # delete orphaned files + removed_files = 0 + error_files = 0 + for file in orphaned_files: + try: + storage.delete(file) + removed_files += 1 + click.echo(click.style(f"- Removing orphaned file: {file}", fg="white")) + except Exception as e: + error_files += 1 + click.echo(click.style(f"- Error deleting orphaned file {file}: {str(e)}", fg="red")) + continue + if error_files == 0: + click.echo(click.style(f"Removed {removed_files} orphaned files without errors.", fg="green")) + else: + click.echo(click.style(f"Removed {removed_files} orphaned files, with {error_files} errors.", fg="yellow")) diff --git a/api/extensions/ext_commands.py b/api/extensions/ext_commands.py index be43f55ea7..ddc2158a02 100644 --- a/api/extensions/ext_commands.py +++ b/api/extensions/ext_commands.py @@ -5,6 +5,7 @@ def init_app(app: DifyApp): from commands import ( add_qdrant_index, clear_free_plan_tenant_expired_logs, + clear_orphaned_file_records, convert_to_agent_apps, create_tenant, extract_plugins, @@ -13,6 +14,7 @@ def init_app(app: DifyApp): install_plugins, migrate_data_for_plugin, old_metadata_migration, + remove_orphaned_files_on_storage, reset_email, reset_encrypt_key_pair, reset_password, @@ -36,6 +38,8 @@ def init_app(app: DifyApp): install_plugins, old_metadata_migration, clear_free_plan_tenant_expired_logs, + clear_orphaned_file_records, + remove_orphaned_files_on_storage, ] for cmd in cmds_to_register: app.cli.add_command(cmd) diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.py index 4c811c66ba..bd35278544 100644 --- a/api/extensions/ext_storage.py +++ b/api/extensions/ext_storage.py @@ -102,6 +102,9 @@ class Storage: def delete(self, filename): return self.storage_runner.delete(filename) + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: + return self.storage_runner.scan(path, files=files, directories=directories) + storage = Storage() diff --git a/api/extensions/storage/base_storage.py b/api/extensions/storage/base_storage.py index 0dedd7ff8c..0393206e54 100644 --- a/api/extensions/storage/base_storage.py +++ b/api/extensions/storage/base_storage.py @@ -30,3 +30,11 @@ class BaseStorage(ABC): @abstractmethod def delete(self, filename): raise NotImplementedError + + def scan(self, path, files=True, directories=False) -> list[str]: + """ + Scan files and directories in the given path. + This method is implemented only in some storage backends. + If a storage backend doesn't support scanning, it will raise NotImplementedError. + """ + raise NotImplementedError("This storage backend doesn't support scanning") diff --git a/api/extensions/storage/opendal_storage.py b/api/extensions/storage/opendal_storage.py index ee8cfa9179..12e2738e9d 100644 --- a/api/extensions/storage/opendal_storage.py +++ b/api/extensions/storage/opendal_storage.py @@ -80,3 +80,20 @@ class OpenDALStorage(BaseStorage): logger.debug(f"file {filename} deleted") return logger.debug(f"file {filename} not found, skip delete") + + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: + if not self.exists(path): + raise FileNotFoundError("Path not found") + + all_files = self.op.scan(path=path) + if files and directories: + logger.debug(f"files and directories on {path} scanned") + return [f.path for f in all_files] + if files: + logger.debug(f"files on {path} scanned") + return [f.path for f in all_files if not f.path.endswith("/")] + elif directories: + logger.debug(f"directories on {path} scanned") + return [f.path for f in all_files if f.path.endswith("/")] + else: + raise ValueError("At least one of files or directories must be True") From c1559a7c8ea1bff27c82f65b83cdab0e96427ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Sun, 27 Apr 2025 11:32:14 +0800 Subject: [PATCH 03/10] fix: LLMResultChunk cause concatenate str and list exception (#18852) --- .../__base/large_language_model.py | 7 +++++-- api/core/model_runtime/utils/helper.py | 17 +++++++++++++++++ api/core/workflow/nodes/llm/node.py | 16 +++------------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/api/core/model_runtime/model_providers/__base/large_language_model.py b/api/core/model_runtime/model_providers/__base/large_language_model.py index 1b799131e7..1a4e4a1537 100644 --- a/api/core/model_runtime/model_providers/__base/large_language_model.py +++ b/api/core/model_runtime/model_providers/__base/large_language_model.py @@ -2,7 +2,7 @@ import logging import time import uuid from collections.abc import Generator, Sequence -from typing import Optional, Union +from typing import Optional, Union, cast from pydantic import ConfigDict @@ -20,6 +20,7 @@ from core.model_runtime.entities.model_entities import ( PriceType, ) from core.model_runtime.model_providers.__base.ai_model import AIModel +from core.model_runtime.utils.helper import convert_llm_result_chunk_to_str from core.plugin.manager.model import PluginModelManager logger = logging.getLogger(__name__) @@ -280,7 +281,9 @@ class LargeLanguageModel(AIModel): callbacks=callbacks, ) - assistant_message.content += chunk.delta.message.content + text = convert_llm_result_chunk_to_str(chunk.delta.message.content) + current_content = cast(str, assistant_message.content) + assistant_message.content = current_content + text real_model = chunk.model if chunk.delta.usage: usage = chunk.delta.usage diff --git a/api/core/model_runtime/utils/helper.py b/api/core/model_runtime/utils/helper.py index 5e8a723ec7..53789a8e91 100644 --- a/api/core/model_runtime/utils/helper.py +++ b/api/core/model_runtime/utils/helper.py @@ -1,6 +1,8 @@ import pydantic from pydantic import BaseModel +from core.model_runtime.entities.message_entities import PromptMessageContentUnionTypes + def dump_model(model: BaseModel) -> dict: if hasattr(pydantic, "model_dump"): @@ -8,3 +10,18 @@ def dump_model(model: BaseModel) -> dict: return pydantic.model_dump(model) # type: ignore else: return model.model_dump() + + +def convert_llm_result_chunk_to_str(content: None | str | list[PromptMessageContentUnionTypes]) -> str: + if content is None: + message_text = "" + elif isinstance(content, str): + message_text = content + elif isinstance(content, list): + # Assuming the list contains PromptMessageContent objects with a "data" attribute + message_text = "".join( + item.data if hasattr(item, "data") and isinstance(item.data, str) else str(item) for item in content + ) + else: + message_text = str(content) + return message_text diff --git a/api/core/workflow/nodes/llm/node.py b/api/core/workflow/nodes/llm/node.py index 1089e7168e..35b146e5d9 100644 --- a/api/core/workflow/nodes/llm/node.py +++ b/api/core/workflow/nodes/llm/node.py @@ -38,6 +38,7 @@ from core.model_runtime.entities.model_entities import ( ) from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.utils.encoders import jsonable_encoder +from core.model_runtime.utils.helper import convert_llm_result_chunk_to_str from core.plugin.entities.plugin import ModelProviderID from core.prompt.entities.advanced_prompt_entities import CompletionModelPromptTemplate, MemoryConfig from core.prompt.utils.prompt_message_util import PromptMessageUtil @@ -269,18 +270,7 @@ class LLMNode(BaseNode[LLMNodeData]): def _handle_invoke_result(self, invoke_result: LLMResult | Generator) -> Generator[NodeEvent, None, None]: if isinstance(invoke_result, LLMResult): - content = invoke_result.message.content - if content is None: - message_text = "" - elif isinstance(content, str): - message_text = content - elif isinstance(content, list): - # Assuming the list contains PromptMessageContent objects with a "data" attribute - message_text = "".join( - item.data if hasattr(item, "data") and isinstance(item.data, str) else str(item) for item in content - ) - else: - message_text = str(content) + message_text = convert_llm_result_chunk_to_str(invoke_result.message.content) yield ModelInvokeCompletedEvent( text=message_text, @@ -295,7 +285,7 @@ class LLMNode(BaseNode[LLMNodeData]): usage = None finish_reason = None for result in invoke_result: - text = result.delta.message.content + text = convert_llm_result_chunk_to_str(result.delta.message.content) full_text += text yield RunStreamChunkEvent(chunk_content=text, from_variable_selector=[self.node_id, "text"]) From 136995d2a1420c9b3b821a4ef8f28f7ca81cb94a Mon Sep 17 00:00:00 2001 From: devxing <66726106+devxing@users.noreply.github.com> Date: Sun, 27 Apr 2025 12:12:46 +0800 Subject: [PATCH 04/10] fix: change delete app status code from 204 to 200 (#18398) Co-authored-by: devxing Co-authored-by: crazywoola <427733928@qq.com> --- api/controllers/console/app/annotation.py | 2 +- api/controllers/console/app/ops_trace.py | 2 +- api/controllers/console/auth/data_source_bearer_auth.py | 2 +- api/controllers/console/datasets/datasets_segments.py | 6 +++--- api/controllers/console/datasets/external.py | 2 +- api/controllers/console/datasets/metadata.py | 2 +- api/controllers/console/explore/installed_app.py | 2 +- api/controllers/console/explore/saved_message.py | 2 +- api/controllers/console/extension.py | 2 +- api/controllers/console/tag/tags.py | 2 +- api/controllers/service_api/app/annotation.py | 2 +- api/controllers/service_api/app/conversation.py | 2 +- api/controllers/service_api/dataset/document.py | 2 +- api/controllers/service_api/dataset/metadata.py | 2 +- api/controllers/service_api/dataset/segment.py | 4 ++-- api/controllers/web/saved_message.py | 2 +- 16 files changed, 19 insertions(+), 19 deletions(-) diff --git a/api/controllers/console/app/annotation.py b/api/controllers/console/app/annotation.py index fcd8ed1882..48353a63af 100644 --- a/api/controllers/console/app/annotation.py +++ b/api/controllers/console/app/annotation.py @@ -186,7 +186,7 @@ class AnnotationUpdateDeleteApi(Resource): app_id = str(app_id) annotation_id = str(annotation_id) AppAnnotationService.delete_app_annotation(app_id, annotation_id) - return {"result": "success"}, 200 + return {"result": "success"}, 204 class AnnotationBatchImportApi(Resource): diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index dd25af8ebf..7176440e16 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -84,7 +84,7 @@ class TraceAppConfigApi(Resource): result = OpsService.delete_tracing_app_config(app_id=app_id, tracing_provider=args["tracing_provider"]) if not result: raise TracingConfigNotExist() - return {"result": "success"} + return {"result": "success"}, 204 except Exception as e: raise BadRequest(str(e)) diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index ea00c2b8c2..5f0762e4a5 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -65,7 +65,7 @@ class ApiKeyAuthDataSourceBindingDelete(Resource): ApiKeyAuthService.delete_provider_auth(current_user.current_tenant_id, binding_id) - return {"result": "success"}, 200 + return {"result": "success"}, 204 api.add_resource(ApiKeyAuthDataSource, "/api-key-auth/data-source") diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 696aaa94db..5c54ecbe81 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -131,7 +131,7 @@ class DatasetDocumentSegmentListApi(Resource): except services.errors.account.NoPermissionError as e: raise Forbidden(str(e)) SegmentService.delete_segments(segment_ids, document, dataset) - return {"result": "success"}, 200 + return {"result": "success"}, 204 class DatasetDocumentSegmentApi(Resource): @@ -333,7 +333,7 @@ class DatasetDocumentSegmentUpdateApi(Resource): except services.errors.account.NoPermissionError as e: raise Forbidden(str(e)) SegmentService.delete_segment(segment, document, dataset) - return {"result": "success"}, 200 + return {"result": "success"}, 204 class DatasetDocumentSegmentBatchImportApi(Resource): @@ -590,7 +590,7 @@ class ChildChunkUpdateApi(Resource): SegmentService.delete_child_chunk(child_chunk, dataset) except ChildChunkDeleteIndexServiceError as e: raise ChildChunkDeleteIndexError(str(e)) - return {"result": "success"}, 200 + return {"result": "success"}, 204 @setup_required @login_required diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.py index 2c031172bf..aee8323f23 100644 --- a/api/controllers/console/datasets/external.py +++ b/api/controllers/console/datasets/external.py @@ -135,7 +135,7 @@ class ExternalApiTemplateApi(Resource): raise Forbidden() ExternalDatasetService.delete_external_knowledge_api(current_user.current_tenant_id, external_knowledge_api_id) - return {"result": "success"}, 200 + return {"result": "success"}, 204 class ExternalApiUseCheckApi(Resource): diff --git a/api/controllers/console/datasets/metadata.py b/api/controllers/console/datasets/metadata.py index fc9711169f..e4cac40ca1 100644 --- a/api/controllers/console/datasets/metadata.py +++ b/api/controllers/console/datasets/metadata.py @@ -82,7 +82,7 @@ class DatasetMetadataApi(Resource): DatasetService.check_dataset_permission(dataset, current_user) MetadataService.delete_metadata(dataset_id_str, metadata_id_str) - return 200 + return {"result": "success"}, 204 class DatasetMetadataBuiltInFieldApi(Resource): diff --git a/api/controllers/console/explore/installed_app.py b/api/controllers/console/explore/installed_app.py index 86550b2bdf..132da11878 100644 --- a/api/controllers/console/explore/installed_app.py +++ b/api/controllers/console/explore/installed_app.py @@ -113,7 +113,7 @@ class InstalledAppApi(InstalledAppResource): db.session.delete(installed_app) db.session.commit() - return {"result": "success", "message": "App uninstalled successfully"} + return {"result": "success", "message": "App uninstalled successfully"}, 204 def patch(self, installed_app): parser = reqparse.RequestParser() diff --git a/api/controllers/console/explore/saved_message.py b/api/controllers/console/explore/saved_message.py index 9f0c496645..3a1655d0ee 100644 --- a/api/controllers/console/explore/saved_message.py +++ b/api/controllers/console/explore/saved_message.py @@ -72,7 +72,7 @@ class SavedMessageApi(InstalledAppResource): SavedMessageService.delete(app_model, current_user, message_id) - return {"result": "success"} + return {"result": "success"}, 204 api.add_resource( diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py index ed6cedb220..833da0d03c 100644 --- a/api/controllers/console/extension.py +++ b/api/controllers/console/extension.py @@ -99,7 +99,7 @@ class APIBasedExtensionDetailAPI(Resource): APIBasedExtensionService.delete(extension_data_from_db) - return {"result": "success"} + return {"result": "success"}, 204 api.add_resource(CodeBasedExtensionAPI, "/code-based-extension") diff --git a/api/controllers/console/tag/tags.py b/api/controllers/console/tag/tags.py index da83f64019..0d0d7ae95f 100644 --- a/api/controllers/console/tag/tags.py +++ b/api/controllers/console/tag/tags.py @@ -86,7 +86,7 @@ class TagUpdateDeleteApi(Resource): TagService.delete_tag(tag_id) - return 200 + return 204 class TagBindingCreateApi(Resource): diff --git a/api/controllers/service_api/app/annotation.py b/api/controllers/service_api/app/annotation.py index cffa3665b1..522a96b791 100644 --- a/api/controllers/service_api/app/annotation.py +++ b/api/controllers/service_api/app/annotation.py @@ -98,7 +98,7 @@ class AnnotationUpdateDeleteApi(Resource): annotation_id = str(annotation_id) AppAnnotationService.delete_app_annotation(app_model.id, annotation_id) - return {"result": "success"}, 200 + return {"result": "success"}, 204 api.add_resource(AnnotationReplyActionApi, "/apps/annotation-reply/") diff --git a/api/controllers/service_api/app/conversation.py b/api/controllers/service_api/app/conversation.py index 55600a3fd0..dfc357e1ab 100644 --- a/api/controllers/service_api/app/conversation.py +++ b/api/controllers/service_api/app/conversation.py @@ -72,7 +72,7 @@ class ConversationDetailApi(Resource): ConversationService.delete(app_model, conversation_id, end_user) except services.errors.conversation.ConversationNotExistsError: raise NotFound("Conversation Not Exists.") - return {"result": "success"}, 200 + return {"result": "success"}, 204 class ConversationRenameApi(Resource): diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py index eec6afc9ef..9e943e2b2d 100644 --- a/api/controllers/service_api/dataset/document.py +++ b/api/controllers/service_api/dataset/document.py @@ -323,7 +323,7 @@ class DocumentDeleteApi(DatasetApiResource): except services.errors.document.DocumentIndexingError: raise DocumentIndexingError("Cannot delete document during indexing.") - return {"result": "success"}, 200 + return {"result": "success"}, 204 class DocumentListApi(DatasetApiResource): diff --git a/api/controllers/service_api/dataset/metadata.py b/api/controllers/service_api/dataset/metadata.py index 298c8a8df8..35578eae54 100644 --- a/api/controllers/service_api/dataset/metadata.py +++ b/api/controllers/service_api/dataset/metadata.py @@ -63,7 +63,7 @@ class DatasetMetadataServiceApi(DatasetApiResource): DatasetService.check_dataset_permission(dataset, current_user) MetadataService.delete_metadata(dataset_id_str, metadata_id_str) - return 200 + return 204 class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource): diff --git a/api/controllers/service_api/dataset/segment.py b/api/controllers/service_api/dataset/segment.py index 2a79e15cc5..95753cfd67 100644 --- a/api/controllers/service_api/dataset/segment.py +++ b/api/controllers/service_api/dataset/segment.py @@ -159,7 +159,7 @@ class DatasetSegmentApi(DatasetApiResource): if not segment: raise NotFound("Segment not found.") SegmentService.delete_segment(segment, document, dataset) - return {"result": "success"}, 200 + return {"result": "success"}, 204 @cloud_edition_billing_resource_check("vector_space", "dataset") def post(self, tenant_id, dataset_id, document_id, segment_id): @@ -344,7 +344,7 @@ class DatasetChildChunkApi(DatasetApiResource): except ChildChunkDeleteIndexServiceError as e: raise ChildChunkDeleteIndexError(str(e)) - return {"result": "success"}, 200 + return {"result": "success"}, 204 @cloud_edition_billing_resource_check("vector_space", "dataset") @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset") diff --git a/api/controllers/web/saved_message.py b/api/controllers/web/saved_message.py index 6a9b818907..ab2d4abcd3 100644 --- a/api/controllers/web/saved_message.py +++ b/api/controllers/web/saved_message.py @@ -67,7 +67,7 @@ class SavedMessageApi(WebApiResource): SavedMessageService.delete(app_model, end_user, message_id) - return {"result": "success"} + return {"result": "success"}, 204 api.add_resource(SavedMessageListApi, "/saved-messages") From bed47dffb9c1f7f9882247b1cd895bb7c98d0a22 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Sun, 27 Apr 2025 14:01:02 +0900 Subject: [PATCH 05/10] fix: update notice for users for clear-orphaned-file-records and remove-orphaned-files-on-storage commands (#18864) --- api/commands.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/api/commands.py b/api/commands.py index f1ff8e1401..3881439ddf 100644 --- a/api/commands.py +++ b/api/commands.py @@ -869,6 +869,15 @@ def clear_orphaned_file_records(): click.echo( click.style("This cannot be undone. Please make sure to back up your database before proceeding.", fg="yellow") ) + click.echo( + click.style( + ( + "It is also recommended to run this during the maintenance window, " + "as this may cause high load on your instance." + ), + fg="yellow", + ) + ) click.confirm("Do you want to proceed?", abort=True) # start the cleanup process @@ -1008,7 +1017,16 @@ def remove_orphaned_files_on_storage(): ) ) click.echo( - click.style("This cannot be undone. Please make sure to back up your database before proceeding.", fg="yellow") + click.style("This cannot be undone. Please make sure to back up your storage before proceeding.", fg="yellow") + ) + click.echo( + click.style( + ( + "It is also recommended to run this during the maintenance window, " + "as this may cause high load on your instance." + ), + fg="yellow", + ) ) click.confirm("Do you want to proceed?", abort=True) From 58a929edd58f0277d05d8f9b018cbf07545c0da7 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Sun, 27 Apr 2025 14:00:35 +0800 Subject: [PATCH 06/10] fix: install plugins permissions (#18870) --- .../install-bundle/steps/install.tsx | 4 +++- web/app/components/plugins/plugin-page/index.tsx | 12 ++++++++++-- .../plugins/plugin-page/use-permission.ts | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/web/app/components/plugins/install-plugin/install-bundle/steps/install.tsx b/web/app/components/plugins/install-plugin/install-bundle/steps/install.tsx index cff1afff02..ee2699b4bb 100644 --- a/web/app/components/plugins/install-plugin/install-bundle/steps/install.tsx +++ b/web/app/components/plugins/install-plugin/install-bundle/steps/install.tsx @@ -8,6 +8,7 @@ import { useTranslation } from 'react-i18next' import InstallMulti from './install-multi' import { useInstallOrUpdate } from '@/service/use-plugins' import useRefreshPluginList from '../../hooks/use-refresh-plugin-list' +import { useCanInstallPluginFromMarketplace } from '@/app/components/plugins/plugin-page/use-permission' const i18nPrefix = 'plugin.installModal' type Props = { @@ -74,6 +75,7 @@ const Install: FC = ({ installedInfo: installedInfo!, }) } + const { canInstallPluginFromMarketplace } = useCanInstallPluginFromMarketplace() return ( <>
@@ -101,7 +103,7 @@ const Install: FC = ({ + +
+ + ) +} + +export default DSLConfirmModal diff --git a/web/app/components/explore/app-list/index.tsx b/web/app/components/explore/app-list/index.tsx index e217dda2b2..7e2d990bc8 100644 --- a/web/app/components/explore/app-list/index.tsx +++ b/web/app/components/explore/app-list/index.tsx @@ -1,12 +1,10 @@ 'use client' -import React, { useMemo, useState } from 'react' -import { useRouter } from 'next/navigation' +import React, { useCallback, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import useSWR from 'swr' import { useDebounceFn } from 'ahooks' -import Toast from '../../base/toast' import s from './style.module.css' import cn from '@/utils/classnames' import ExploreContext from '@/context/explore-context' @@ -14,17 +12,16 @@ import type { App } from '@/models/explore' import Category from '@/app/components/explore/category' import AppCard from '@/app/components/explore/app-card' import { fetchAppDetail, fetchAppList } from '@/service/explore' -import { importDSL } from '@/service/apps' import { useTabSearchParams } from '@/hooks/use-tab-searchparams' import CreateAppModal from '@/app/components/explore/create-app-modal' import type { CreateAppModalProps } from '@/app/components/explore/create-app-modal' import Loading from '@/app/components/base/loading' -import { NEED_REFRESH_APP_LIST_KEY } from '@/config' -import { useAppContext } from '@/context/app-context' -import { getRedirection } from '@/utils/app-redirection' import Input from '@/app/components/base/input' -import { DSLImportMode } from '@/models/app' -import { usePluginDependencies } from '@/app/components/workflow/plugin-dependency/hooks' +import { + DSLImportMode, +} from '@/models/app' +import { useImportDSL } from '@/hooks/use-import-dsl' +import DSLConfirmModal from '@/app/components/app/create-from-dsl-modal/dsl-confirm-modal' type AppsProps = { onSuccess?: () => void @@ -39,8 +36,6 @@ const Apps = ({ onSuccess, }: AppsProps) => { const { t } = useTranslation() - const { isCurrentWorkspaceEditor } = useAppContext() - const { push } = useRouter() const { hasEditPermission } = useContext(ExploreContext) const allCategoriesEn = t('explore.apps.allCategories', { lng: 'en' }) @@ -115,7 +110,14 @@ const Apps = ({ const [currApp, setCurrApp] = React.useState(null) const [isShowCreateModal, setIsShowCreateModal] = React.useState(false) - const { handleCheckPluginDependencies } = usePluginDependencies() + + const { + handleImportDSL, + handleImportDSLConfirm, + versions, + isFetching, + } = useImportDSL() + const [showDSLConfirmModal, setShowDSLConfirmModal] = useState(false) const onCreate: CreateAppModalProps['onConfirm'] = async ({ name, icon_type, @@ -123,36 +125,34 @@ const Apps = ({ icon_background, description, }) => { - const { export_data, mode } = await fetchAppDetail( + const { export_data } = await fetchAppDetail( currApp?.app.id as string, ) - try { - const app = await importDSL({ - mode: DSLImportMode.YAML_CONTENT, - yaml_content: export_data, - name, - icon_type, - icon, - icon_background, - description, - }) - setIsShowCreateModal(false) - Toast.notify({ - type: 'success', - message: t('app.newApp.appCreated'), - }) - if (onSuccess) - onSuccess() - if (app.app_id) - await handleCheckPluginDependencies(app.app_id) - localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1') - getRedirection(isCurrentWorkspaceEditor, { id: app.app_id!, mode }, push) - } - catch { - Toast.notify({ type: 'error', message: t('app.newApp.appCreateFailed') }) + const payload = { + mode: DSLImportMode.YAML_CONTENT, + yaml_content: export_data, + name, + icon_type, + icon, + icon_background, + description, } + await handleImportDSL(payload, { + onSuccess: () => { + setIsShowCreateModal(false) + }, + onPending: () => { + setShowDSLConfirmModal(true) + }, + }) } + const onConfirmDSL = useCallback(async () => { + await handleImportDSLConfirm({ + onSuccess, + }) + }, [handleImportDSLConfirm, onSuccess]) + if (!categories || categories.length === 0) { return (
@@ -225,9 +225,20 @@ const Apps = ({ appDescription={currApp?.app.description || ''} show={isShowCreateModal} onConfirm={onCreate} + confirmDisabled={isFetching} onHide={() => setIsShowCreateModal(false)} /> )} + { + showDSLConfirmModal && ( + setShowDSLConfirmModal(false)} + onConfirm={onConfirmDSL} + confirmDisabled={isFetching} + /> + ) + }
) } diff --git a/web/app/components/explore/create-app-modal/index.tsx b/web/app/components/explore/create-app-modal/index.tsx index d6d521833a..f30b286786 100644 --- a/web/app/components/explore/create-app-modal/index.tsx +++ b/web/app/components/explore/create-app-modal/index.tsx @@ -35,6 +35,7 @@ export type CreateAppModalProps = { description: string use_icon_as_answer_icon?: boolean }) => Promise + confirmDisabled?: boolean onHide: () => void } @@ -50,6 +51,7 @@ const CreateAppModal = ({ appMode, appUseIconAsAnswerIcon, onConfirm, + confirmDisabled, onHide, }: CreateAppModalProps) => { const { t } = useTranslation() @@ -160,7 +162,7 @@ const CreateAppModal = ({