Merge remote-tracking branch 'upstream/main' into feat/support-image-generate-for-gemini

api/commands.py

@@ -17,6 +17,7 @@ from core.rag.models.document import Document
 from events.app_event import app_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
+from extensions.ext_storage import storage
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
@@ -815,3 +816,274 @@ def clear_free_plan_tenant_expired_logs(days: int, batch: int, tenant_ids: list[
     ClearFreePlanTenantExpiredLogs.process(days, batch, tenant_ids)
 
     click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green"))
+
+
+@click.command("clear-orphaned-file-records", help="Clear orphaned file records.")
+def clear_orphaned_file_records():
+    """
+    Clear orphaned file records in the database.
+    """
+
+    # define tables and columns to process
+    files_tables = [
+        {"table": "upload_files", "id_column": "id", "key_column": "key"},
+        {"table": "tool_files", "id_column": "id", "key_column": "file_key"},
+    ]
+    ids_tables = [
+        {"type": "uuid", "table": "message_files", "column": "upload_file_id"},
+        {"type": "text", "table": "documents", "column": "data_source_info"},
+        {"type": "text", "table": "document_segments", "column": "content"},
+        {"type": "text", "table": "messages", "column": "answer"},
+        {"type": "text", "table": "workflow_node_executions", "column": "inputs"},
+        {"type": "text", "table": "workflow_node_executions", "column": "process_data"},
+        {"type": "text", "table": "workflow_node_executions", "column": "outputs"},
+        {"type": "text", "table": "conversations", "column": "introduction"},
+        {"type": "text", "table": "conversations", "column": "system_instruction"},
+        {"type": "json", "table": "messages", "column": "inputs"},
+        {"type": "json", "table": "messages", "column": "message"},
+    ]
+
+    # notify user and ask for confirmation
+    click.echo(
+        click.style("This command will find and delete orphaned file records in the following tables:", fg="yellow")
+    )
+    for files_table in files_tables:
+        click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
+    click.echo(
+        click.style("The following tables and columns will be scanned to find orphaned file records:", fg="yellow")
+    )
+    for ids_table in ids_tables:
+        click.echo(click.style(f"- {ids_table['table']} ({ids_table['column']})", fg="yellow"))
+    click.echo("")
+
+    click.echo(click.style("!!! USE WITH CAUTION !!!", fg="red"))
+    click.echo(
+        click.style(
+            (
+                "Since not all patterns have been fully tested, "
+                "please note that this command may delete unintended file records."
+            ),
+            fg="yellow",
+        )
+    )
+    click.echo(
+        click.style("This cannot be undone. Please make sure to back up your database before proceeding.", fg="yellow")
+    )
+    click.echo(
+        click.style(
+            (
+                "It is also recommended to run this during the maintenance window, "
+                "as this may cause high load on your instance."
+            ),
+            fg="yellow",
+        )
+    )
+    click.confirm("Do you want to proceed?", abort=True)
+
+    # start the cleanup process
+    click.echo(click.style("Starting orphaned file records cleanup.", fg="white"))
+
+    try:
+        # fetch file id and keys from each table
+        all_files_in_tables = []
+        for files_table in files_tables:
+            click.echo(click.style(f"- Listing file records in table {files_table['table']}", fg="white"))
+            query = f"SELECT {files_table['id_column']}, {files_table['key_column']} FROM {files_table['table']}"
+            with db.engine.begin() as conn:
+                rs = conn.execute(db.text(query))
+            for i in rs:
+                all_files_in_tables.append({"table": files_table["table"], "id": str(i[0]), "key": i[1]})
+        click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white"))
+
+        # fetch referred table and columns
+        guid_regexp = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+        all_ids_in_tables = []
+        for ids_table in ids_tables:
+            query = ""
+            if ids_table["type"] == "uuid":
+                click.echo(
+                    click.style(
+                        f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}", fg="white"
+                    )
+                )
+                query = (
+                    f"SELECT {ids_table['column']} FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(db.text(query))
+                for i in rs:
+                    all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
+            elif ids_table["type"] == "text":
+                click.echo(
+                    click.style(
+                        f"- Listing file-id-like strings in column {ids_table['column']} in table {ids_table['table']}",
+                        fg="white",
+                    )
+                )
+                query = (
+                    f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id "
+                    f"FROM {ids_table['table']}"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(db.text(query))
+                for i in rs:
+                    for j in i[0]:
+                        all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+            elif ids_table["type"] == "json":
+                click.echo(
+                    click.style(
+                        (
+                            f"- Listing file-id-like JSON string in column {ids_table['column']} "
+                            f"in table {ids_table['table']}"
+                        ),
+                        fg="white",
+                    )
+                )
+                query = (
+                    f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id "
+                    f"FROM {ids_table['table']}"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(db.text(query))
+                for i in rs:
+                    for j in i[0]:
+                        all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+        click.echo(click.style(f"Found {len(all_ids_in_tables)} file ids in tables.", fg="white"))
+
+    except Exception as e:
+        click.echo(click.style(f"Error fetching keys: {str(e)}", fg="red"))
+        return
+
+    # find orphaned files
+    all_files = [file["id"] for file in all_files_in_tables]
+    all_ids = [file["id"] for file in all_ids_in_tables]
+    orphaned_files = list(set(all_files) - set(all_ids))
+    if not orphaned_files:
+        click.echo(click.style("No orphaned file records found. There is nothing to delete.", fg="green"))
+        return
+    click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white"))
+    for file in orphaned_files:
+        click.echo(click.style(f"- orphaned file id: {file}", fg="black"))
+    click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
+
+    # delete orphaned records for each file
+    try:
+        for files_table in files_tables:
+            click.echo(click.style(f"- Deleting orphaned file records in table {files_table['table']}", fg="white"))
+            query = f"DELETE FROM {files_table['table']} WHERE {files_table['id_column']} IN :ids"
+            with db.engine.begin() as conn:
+                conn.execute(db.text(query), {"ids": tuple(orphaned_files)})
+    except Exception as e:
+        click.echo(click.style(f"Error deleting orphaned file records: {str(e)}", fg="red"))
+        return
+    click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green"))
+
+
+@click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.")
+def remove_orphaned_files_on_storage():
+    """
+    Remove orphaned files on the storage.
+    """
+
+    # define tables and columns to process
+    files_tables = [
+        {"table": "upload_files", "key_column": "key"},
+        {"table": "tool_files", "key_column": "file_key"},
+    ]
+    storage_paths = ["image_files", "tools", "upload_files"]
+
+    # notify user and ask for confirmation
+    click.echo(click.style("This command will find and remove orphaned files on the storage,", fg="yellow"))
+    click.echo(
+        click.style("by comparing the files on the storage with the records in the following tables:", fg="yellow")
+    )
+    for files_table in files_tables:
+        click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
+    click.echo(click.style("The following paths on the storage will be scanned to find orphaned files:", fg="yellow"))
+    for storage_path in storage_paths:
+        click.echo(click.style(f"- {storage_path}", fg="yellow"))
+    click.echo("")
+
+    click.echo(click.style("!!! USE WITH CAUTION !!!", fg="red"))
+    click.echo(
+        click.style(
+            "Currently, this command will work only for opendal based storage (STORAGE_TYPE=opendal).", fg="yellow"
+        )
+    )
+    click.echo(
+        click.style(
+            "Since not all patterns have been fully tested, please note that this command may delete unintended files.",
+            fg="yellow",
+        )
+    )
+    click.echo(
+        click.style("This cannot be undone. Please make sure to back up your storage before proceeding.", fg="yellow")
+    )
+    click.echo(
+        click.style(
+            (
+                "It is also recommended to run this during the maintenance window, "
+                "as this may cause high load on your instance."
+            ),
+            fg="yellow",
+        )
+    )
+    click.confirm("Do you want to proceed?", abort=True)
+
+    # start the cleanup process
+    click.echo(click.style("Starting orphaned files cleanup.", fg="white"))
+
+    # fetch file id and keys from each table
+    all_files_in_tables = []
+    try:
+        for files_table in files_tables:
+            click.echo(click.style(f"- Listing files from table {files_table['table']}", fg="white"))
+            query = f"SELECT {files_table['key_column']} FROM {files_table['table']}"
+            with db.engine.begin() as conn:
+                rs = conn.execute(db.text(query))
+            for i in rs:
+                all_files_in_tables.append(str(i[0]))
+        click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white"))
+    except Exception as e:
+        click.echo(click.style(f"Error fetching keys: {str(e)}", fg="red"))
+
+    all_files_on_storage = []
+    for storage_path in storage_paths:
+        try:
+            click.echo(click.style(f"- Scanning files on storage path {storage_path}", fg="white"))
+            files = storage.scan(path=storage_path, files=True, directories=False)
+            all_files_on_storage.extend(files)
+        except FileNotFoundError as e:
+            click.echo(click.style(f"  -> Skipping path {storage_path} as it does not exist.", fg="yellow"))
+            continue
+        except Exception as e:
+            click.echo(click.style(f"  -> Error scanning files on storage path {storage_path}: {str(e)}", fg="red"))
+            continue
+    click.echo(click.style(f"Found {len(all_files_on_storage)} files on storage.", fg="white"))
+
+    # find orphaned files
+    orphaned_files = list(set(all_files_on_storage) - set(all_files_in_tables))
+    if not orphaned_files:
+        click.echo(click.style("No orphaned files found. There is nothing to remove.", fg="green"))
+        return
+    click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white"))
+    for file in orphaned_files:
+        click.echo(click.style(f"- orphaned file: {file}", fg="black"))
+    click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
+
+    # delete orphaned files
+    removed_files = 0
+    error_files = 0
+    for file in orphaned_files:
+        try:
+            storage.delete(file)
+            removed_files += 1
+            click.echo(click.style(f"- Removing orphaned file: {file}", fg="white"))
+        except Exception as e:
+            error_files += 1
+            click.echo(click.style(f"- Error deleting orphaned file {file}: {str(e)}", fg="red"))
+            continue
+    if error_files == 0:
+        click.echo(click.style(f"Removed {removed_files} orphaned files without errors.", fg="green"))
+    else:
+        click.echo(click.style(f"Removed {removed_files} orphaned files, with {error_files} errors.", fg="yellow"))

api/controllers/console/app/annotation.py

@@ -186,7 +186,7 @@ class AnnotationUpdateDeleteApi(Resource):
         app_id = str(app_id)
         annotation_id = str(annotation_id)
         AppAnnotationService.delete_app_annotation(app_id, annotation_id)
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 class AnnotationBatchImportApi(Resource):

api/controllers/console/app/ops_trace.py

@@ -84,7 +84,7 @@ class TraceAppConfigApi(Resource):
             result = OpsService.delete_tracing_app_config(app_id=app_id, tracing_provider=args["tracing_provider"])
             if not result:
                 raise TracingConfigNotExist()
-            return {"result": "success"}
+            return {"result": "success"}, 204
         except Exception as e:
             raise BadRequest(str(e))
 

api/controllers/console/auth/data_source_bearer_auth.py

@@ -65,7 +65,7 @@ class ApiKeyAuthDataSourceBindingDelete(Resource):
 
         ApiKeyAuthService.delete_provider_auth(current_user.current_tenant_id, binding_id)
 
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 api.add_resource(ApiKeyAuthDataSource, "/api-key-auth/data-source")

api/controllers/console/datasets/datasets_document.py

@@ -40,7 +40,7 @@ from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client

api/controllers/console/datasets/datasets_segments.py

@@ -131,7 +131,7 @@ class DatasetDocumentSegmentListApi(Resource):
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         SegmentService.delete_segments(segment_ids, document, dataset)
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 class DatasetDocumentSegmentApi(Resource):
@@ -333,7 +333,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         SegmentService.delete_segment(segment, document, dataset)
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 class DatasetDocumentSegmentBatchImportApi(Resource):
@@ -590,7 +590,7 @@ class ChildChunkUpdateApi(Resource):
             SegmentService.delete_child_chunk(child_chunk, dataset)
         except ChildChunkDeleteIndexServiceError as e:
             raise ChildChunkDeleteIndexError(str(e))
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
     @setup_required
     @login_required

api/controllers/console/datasets/external.py

@@ -135,7 +135,7 @@ class ExternalApiTemplateApi(Resource):
             raise Forbidden()
 
         ExternalDatasetService.delete_external_knowledge_api(current_user.current_tenant_id, external_knowledge_api_id)
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 class ExternalApiUseCheckApi(Resource):

api/controllers/console/datasets/metadata.py

@@ -82,7 +82,7 @@ class DatasetMetadataApi(Resource):
         DatasetService.check_dataset_permission(dataset, current_user)
 
         MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
-        return 200
+        return {"result": "success"}, 204
 
 
 class DatasetMetadataBuiltInFieldApi(Resource):

api/controllers/console/explore/installed_app.py

@@ -113,7 +113,7 @@ class InstalledAppApi(InstalledAppResource):
         db.session.delete(installed_app)
         db.session.commit()
 
-        return {"result": "success", "message": "App uninstalled successfully"}
+        return {"result": "success", "message": "App uninstalled successfully"}, 204
 
     def patch(self, installed_app):
         parser = reqparse.RequestParser()

api/controllers/console/explore/saved_message.py

@@ -72,7 +72,7 @@ class SavedMessageApi(InstalledAppResource):
 
         SavedMessageService.delete(app_model, current_user, message_id)
 
-        return {"result": "success"}
+        return {"result": "success"}, 204
 
 
 api.add_resource(

api/controllers/console/extension.py

@@ -99,7 +99,7 @@ class APIBasedExtensionDetailAPI(Resource):
 
         APIBasedExtensionService.delete(extension_data_from_db)
 
-        return {"result": "success"}
+        return {"result": "success"}, 204
 
 
 api.add_resource(CodeBasedExtensionAPI, "/code-based-extension")

api/controllers/console/tag/tags.py

@@ -86,7 +86,7 @@ class TagUpdateDeleteApi(Resource):
 
         TagService.delete_tag(tag_id)
 
-        return 200
+        return 204
 
 
 class TagBindingCreateApi(Resource):

api/controllers/console/workspace/endpoint.py

@@ -5,7 +5,7 @@ from werkzeug.exceptions import Forbidden
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.plugin.manager.exc import PluginPermissionDeniedError
+from core.plugin.impl.exc import PluginPermissionDeniedError
 from libs.login import login_required
 from services.plugin.endpoint_service import EndpointService
 

api/controllers/console/workspace/plugin.py

@@ -10,7 +10,7 @@ from controllers.console import api
 from controllers.console.workspace import plugin_permission_required
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
 from libs.login import login_required
 from models.account import TenantPluginPermission
 from services.plugin.plugin_permission_service import PluginPermissionService

api/controllers/service_api/app/annotation.py

@@ -79,7 +79,7 @@ class AnnotationListApi(Resource):
 class AnnotationUpdateDeleteApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
     @marshal_with(annotation_fields)
-    def post(self, app_model: App, end_user: EndUser, annotation_id):
+    def put(self, app_model: App, end_user: EndUser, annotation_id):
         if not current_user.is_editor:
             raise Forbidden()
 
@@ -98,7 +98,7 @@ class AnnotationUpdateDeleteApi(Resource):
 
         annotation_id = str(annotation_id)
         AppAnnotationService.delete_app_annotation(app_model.id, annotation_id)
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 api.add_resource(AnnotationReplyActionApi, "/apps/annotation-reply/<string:action>")

api/controllers/service_api/app/conversation.py

@@ -72,7 +72,7 @@ class ConversationDetailApi(Resource):
             ConversationService.delete(app_model, conversation_id, end_user)
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 class ConversationRenameApi(Resource):

api/controllers/service_api/dataset/document.py

@@ -323,7 +323,7 @@ class DocumentDeleteApi(DatasetApiResource):
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
 
 class DocumentListApi(DatasetApiResource):

api/controllers/service_api/dataset/metadata.py

@@ -63,7 +63,7 @@ class DatasetMetadataServiceApi(DatasetApiResource):
         DatasetService.check_dataset_permission(dataset, current_user)
 
         MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
-        return 200
+        return 204
 
 
 class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource):

api/controllers/service_api/dataset/segment.py

@@ -159,7 +159,7 @@ class DatasetSegmentApi(DatasetApiResource):
         if not segment:
             raise NotFound("Segment not found.")
         SegmentService.delete_segment(segment, document, dataset)
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
     @cloud_edition_billing_resource_check("vector_space", "dataset")
     def post(self, tenant_id, dataset_id, document_id, segment_id):
@@ -344,7 +344,7 @@ class DatasetChildChunkApi(DatasetApiResource):
         except ChildChunkDeleteIndexServiceError as e:
             raise ChildChunkDeleteIndexError(str(e))
 
-        return {"result": "success"}, 200
+        return {"result": "success"}, 204
 
     @cloud_edition_billing_resource_check("vector_space", "dataset")
     @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")

api/controllers/web/saved_message.py

@@ -67,7 +67,7 @@ class SavedMessageApi(WebApiResource):
 
         SavedMessageService.delete(app_model, end_user, message_id)
 
-        return {"result": "success"}
+        return {"result": "success"}, 204
 
 
 api.add_resource(SavedMessageListApi, "/saved-messages")

api/core/agent/strategy/plugin.py

@@ -4,7 +4,7 @@ from typing import Any, Optional
 from core.agent.entities import AgentInvokeMessage
 from core.agent.plugin_entities import AgentStrategyEntity, AgentStrategyParameter
 from core.agent.strategy.base import BaseAgentStrategy
-from core.plugin.manager.agent import PluginAgentManager
+from core.plugin.impl.agent import PluginAgentClient
 from core.plugin.utils.converter import convert_parameters_to_plugin_format
 
 
@@ -42,7 +42,7 @@ class PluginAgentStrategy(BaseAgentStrategy):
         """
         Invoke the agent strategy.
         """
-        manager = PluginAgentManager()
+        manager = PluginAgentClient()
 
         initialized_params = self.initialize_parameters(params)
         params = convert_parameters_to_plugin_format(initialized_params)

api/core/model_runtime/model_providers/__base/ai_model.py

@@ -26,7 +26,7 @@ from core.model_runtime.errors.invoke import (
 )
 from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
 from core.plugin.entities.plugin_daemon import PluginDaemonInnerError, PluginModelProviderEntity
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 
 class AIModel(BaseModel):
@@ -141,7 +141,7 @@ class AIModel(BaseModel):
         :param credentials: model credentials
         :return: model schema
         """
-        plugin_model_manager = PluginModelManager()
+        plugin_model_manager = PluginModelClient()
         cache_key = f"{self.tenant_id}:{self.plugin_id}:{self.provider_name}:{self.model_type.value}:{model}"
         # sort credentials
         sorted_credentials = sorted(credentials.items()) if credentials else []

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -22,7 +22,7 @@ from core.model_runtime.entities.model_entities import (
     PriceType,
 )
 from core.model_runtime.model_providers.__base.ai_model import AIModel
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 logger = logging.getLogger(__name__)
 
@@ -142,7 +142,7 @@ class LargeLanguageModel(AIModel):
         result: Union[LLMResult, Generator[LLMResultChunk, None, None]]
 
         try:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             result = plugin_model_manager.invoke_llm(
                 tenant_id=self.tenant_id,
                 user_id=user or "unknown",
@@ -261,6 +261,16 @@ class LargeLanguageModel(AIModel):
         system_fingerprint = None
         real_model = model
 
+        def _update_message_content(content: str | list[PromptMessageContentUnionTypes] | None):
+            if not content:
+                return
+            if isinstance(content, list):
+                message_content.extend(content)
+                return
+            if isinstance(content, str):
+                message_content.append(TextPromptMessageContent(data=content))
+                return
+
         try:
             for chunk in result:
                 # Following https://github.com/langgenius/dify/issues/17799,
@@ -282,10 +292,8 @@ class LargeLanguageModel(AIModel):
                     callbacks=callbacks,
                 )
 
-                if isinstance(chunk.delta.message.content, list):
+                _update_message_content(chunk.delta.message.content)
-                    message_content.extend(chunk.delta.message.content)
+
-                elif isinstance(chunk.delta.message.content, str):
-                    message_content.append(TextPromptMessageContent(data=chunk.delta.message.content))
                 real_model = chunk.model
                 if chunk.delta.usage:
                     usage = chunk.delta.usage
@@ -332,7 +340,7 @@ class LargeLanguageModel(AIModel):
         :return:
         """
         if dify_config.PLUGIN_BASED_TOKEN_COUNTING_ENABLED:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             return plugin_model_manager.get_llm_num_tokens(
                 tenant_id=self.tenant_id,
                 user_id="unknown",

api/core/model_runtime/model_providers/__base/moderation_model.py

@@ -5,7 +5,7 @@ from pydantic import ConfigDict
 
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 
 class ModerationModel(AIModel):
@@ -31,7 +31,7 @@ class ModerationModel(AIModel):
         self.started_at = time.perf_counter()
 
         try:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             return plugin_model_manager.invoke_moderation(
                 tenant_id=self.tenant_id,
                 user_id=user or "unknown",

api/core/model_runtime/model_providers/__base/rerank_model.py

@@ -3,7 +3,7 @@ from typing import Optional
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.entities.rerank_entities import RerankResult
 from core.model_runtime.model_providers.__base.ai_model import AIModel
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 
 class RerankModel(AIModel):
@@ -36,7 +36,7 @@ class RerankModel(AIModel):
         :return: rerank result
         """
         try:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             return plugin_model_manager.invoke_rerank(
                 tenant_id=self.tenant_id,
                 user_id=user or "unknown",

api/core/model_runtime/model_providers/__base/speech2text_model.py

@@ -4,7 +4,7 @@ from pydantic import ConfigDict
 
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 
 class Speech2TextModel(AIModel):
@@ -28,7 +28,7 @@ class Speech2TextModel(AIModel):
         :return: text for given audio file
         """
         try:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             return plugin_model_manager.invoke_speech_to_text(
                 tenant_id=self.tenant_id,
                 user_id=user or "unknown",

api/core/model_runtime/model_providers/__base/text_embedding_model.py

@@ -6,7 +6,7 @@ from core.entities.embedding_type import EmbeddingInputType
 from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
 from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
 from core.model_runtime.model_providers.__base.ai_model import AIModel
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 
 class TextEmbeddingModel(AIModel):
@@ -38,7 +38,7 @@ class TextEmbeddingModel(AIModel):
         :return: embeddings result
         """
         try:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             return plugin_model_manager.invoke_text_embedding(
                 tenant_id=self.tenant_id,
                 user_id=user or "unknown",
@@ -61,7 +61,7 @@ class TextEmbeddingModel(AIModel):
         :param texts: texts to embed
         :return:
         """
-        plugin_model_manager = PluginModelManager()
+        plugin_model_manager = PluginModelClient()
         return plugin_model_manager.get_text_embedding_num_tokens(
             tenant_id=self.tenant_id,
             user_id="unknown",

api/core/model_runtime/model_providers/__base/tts_model.py

@@ -6,7 +6,7 @@ from pydantic import ConfigDict
 
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +42,7 @@ class TTSModel(AIModel):
         :return: translated audio file
         """
         try:
-            plugin_model_manager = PluginModelManager()
+            plugin_model_manager = PluginModelClient()
             return plugin_model_manager.invoke_tts(
                 tenant_id=self.tenant_id,
                 user_id=user or "unknown",
@@ -65,7 +65,7 @@ class TTSModel(AIModel):
         :param credentials: The credentials required to access the TTS model.
         :return: A list of voices supported by the TTS model.
         """
-        plugin_model_manager = PluginModelManager()
+        plugin_model_manager = PluginModelClient()
         return plugin_model_manager.get_tts_model_voices(
             tenant_id=self.tenant_id,
             user_id="unknown",

api/core/model_runtime/model_providers/model_provider_factory.py

@@ -22,8 +22,8 @@ from core.model_runtime.schema_validators.model_credential_schema_validator impo
 from core.model_runtime.schema_validators.provider_credential_schema_validator import ProviderCredentialSchemaValidator
 from core.plugin.entities.plugin import ModelProviderID
 from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
-from core.plugin.manager.asset import PluginAssetManager
+from core.plugin.impl.asset import PluginAssetManager
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +40,7 @@ class ModelProviderFactory:
         self.provider_position_map = {}
 
         self.tenant_id = tenant_id
-        self.plugin_model_manager = PluginModelManager()
+        self.plugin_model_manager = PluginModelClient()
 
         if not self.provider_position_map:
             # get the path of current classes

api/core/ops/entities/config_entity.py

@@ -7,6 +7,7 @@ class TracingProviderEnum(Enum):
     LANGFUSE = "langfuse"
     LANGSMITH = "langsmith"
     OPIK = "opik"
+    WEAVE = "weave"
 
 
 class BaseTracingConfig(BaseModel):
@@ -88,5 +89,26 @@ class OpikConfig(BaseTracingConfig):
         return v
 
 
+class WeaveConfig(BaseTracingConfig):
+    """
+    Model class for Weave tracing config.
+    """
+
+    api_key: str
+    entity: str | None = None
+    project: str
+    endpoint: str = "https://trace.wandb.ai"
+
+    @field_validator("endpoint")
+    @classmethod
+    def set_value(cls, v, info: ValidationInfo):
+        if v is None or v == "":
+            v = "https://trace.wandb.ai"
+        if not v.startswith("https://"):
+            raise ValueError("endpoint must start with https://")
+
+        return v
+
+
 OPS_FILE_PATH = "ops_trace/"
 OPS_TRACE_FAILED_KEY = "FAILED_OPS_TRACE"

api/core/ops/ops_trace_manager.py

@@ -20,6 +20,7 @@ from core.ops.entities.config_entity import (
     LangSmithConfig,
     OpikConfig,
     TracingProviderEnum,
+    WeaveConfig,
 )
 from core.ops.entities.trace_entity import (
     DatasetRetrievalTraceInfo,
@@ -34,7 +35,9 @@ from core.ops.entities.trace_entity import (
 )
 from core.ops.langfuse_trace.langfuse_trace import LangFuseDataTrace
 from core.ops.langsmith_trace.langsmith_trace import LangSmithDataTrace
+from core.ops.opik_trace.opik_trace import OpikDataTrace
 from core.ops.utils import get_message_data
+from core.ops.weave_trace.weave_trace import WeaveDataTrace
 from extensions.ext_database import db
 from extensions.ext_storage import storage
 from models.model import App, AppModelConfig, Conversation, Message, MessageFile, TraceAppConfig
@@ -43,8 +46,6 @@ from tasks.ops_trace_task import process_trace_tasks
 
 
 def build_opik_trace_instance(config: OpikConfig):
-    from core.ops.opik_trace.opik_trace import OpikDataTrace
-
     return OpikDataTrace(config)
 
 
@@ -67,6 +68,12 @@ provider_config_map: dict[str, dict[str, Any]] = {
         "other_keys": ["project", "url", "workspace"],
         "trace_instance": lambda config: build_opik_trace_instance(config),
     },
+    TracingProviderEnum.WEAVE.value: {
+        "config_class": WeaveConfig,
+        "secret_keys": ["api_key"],
+        "other_keys": ["project", "entity", "endpoint"],
+        "trace_instance": WeaveDataTrace,
+    },
 }
 
 

api/core/ops/weave_trace/entities/weave_trace_entity.py

@@ -0,0 +1,97 @@
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel, Field, field_validator
+from pydantic_core.core_schema import ValidationInfo
+
+from core.ops.utils import replace_text_with_content
+
+
+class WeaveTokenUsage(BaseModel):
+    input_tokens: Optional[int] = None
+    output_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+
+
+class WeaveMultiModel(BaseModel):
+    file_list: Optional[list[str]] = Field(None, description="List of files")
+
+
+class WeaveTraceModel(WeaveTokenUsage, WeaveMultiModel):
+    id: str = Field(..., description="ID of the trace")
+    op: str = Field(..., description="Name of the operation")
+    inputs: Optional[Union[str, dict[str, Any], list, None]] = Field(None, description="Inputs of the trace")
+    outputs: Optional[Union[str, dict[str, Any], list, None]] = Field(None, description="Outputs of the trace")
+    attributes: Optional[Union[str, dict[str, Any], list, None]] = Field(
+        None, description="Metadata and attributes associated with trace"
+    )
+    exception: Optional[str] = Field(None, description="Exception message of the trace")
+
+    @field_validator("inputs", "outputs")
+    @classmethod
+    def ensure_dict(cls, v, info: ValidationInfo):
+        field_name = info.field_name
+        values = info.data
+        if v == {} or v is None:
+            return v
+        usage_metadata = {
+            "input_tokens": values.get("input_tokens", 0),
+            "output_tokens": values.get("output_tokens", 0),
+            "total_tokens": values.get("total_tokens", 0),
+        }
+        file_list = values.get("file_list", [])
+        if isinstance(v, str):
+            if field_name == "inputs":
+                return {
+                    "messages": {
+                        "role": "user",
+                        "content": v,
+                        "usage_metadata": usage_metadata,
+                        "file_list": file_list,
+                    },
+                }
+            elif field_name == "outputs":
+                return {
+                    "choices": {
+                        "role": "ai",
+                        "content": v,
+                        "usage_metadata": usage_metadata,
+                        "file_list": file_list,
+                    },
+                }
+        elif isinstance(v, list):
+            data = {}
+            if len(v) > 0 and isinstance(v[0], dict):
+                # rename text to content
+                v = replace_text_with_content(data=v)
+                if field_name == "inputs":
+                    data = {
+                        "messages": [
+                            dict(msg, **{"usage_metadata": usage_metadata, "file_list": file_list}) for msg in v
+                        ]
+                        if isinstance(v, list)
+                        else v,
+                    }
+                elif field_name == "outputs":
+                    data = {
+                        "choices": {
+                            "role": "ai",
+                            "content": v,
+                            "usage_metadata": usage_metadata,
+                            "file_list": file_list,
+                        },
+                    }
+                return data
+            else:
+                return {
+                    "choices": {
+                        "role": "ai" if field_name == "outputs" else "user",
+                        "content": str(v),
+                        "usage_metadata": usage_metadata,
+                        "file_list": file_list,
+                    },
+                }
+        if isinstance(v, dict):
+            v["usage_metadata"] = usage_metadata
+            v["file_list"] = file_list
+            return v
+        return v

api/core/ops/weave_trace/weave_trace.py

@@ -0,0 +1,420 @@
+import json
+import logging
+import os
+import uuid
+from datetime import datetime, timedelta
+from typing import Any, Optional, cast
+
+import wandb
+import weave
+
+from core.ops.base_trace_instance import BaseTraceInstance
+from core.ops.entities.config_entity import WeaveConfig
+from core.ops.entities.trace_entity import (
+    BaseTraceInfo,
+    DatasetRetrievalTraceInfo,
+    GenerateNameTraceInfo,
+    MessageTraceInfo,
+    ModerationTraceInfo,
+    SuggestedQuestionTraceInfo,
+    ToolTraceInfo,
+    TraceTaskName,
+    WorkflowTraceInfo,
+)
+from core.ops.weave_trace.entities.weave_trace_entity import WeaveTraceModel
+from extensions.ext_database import db
+from models.model import EndUser, MessageFile
+from models.workflow import WorkflowNodeExecution
+
+logger = logging.getLogger(__name__)
+
+
+class WeaveDataTrace(BaseTraceInstance):
+    def __init__(
+        self,
+        weave_config: WeaveConfig,
+    ):
+        super().__init__(weave_config)
+        self.weave_api_key = weave_config.api_key
+        self.project_name = weave_config.project
+        self.entity = weave_config.entity
+
+        # Login with API key first
+        login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True)
+        if not login_status:
+            logger.error("Failed to login to Weights & Biases with the provided API key")
+            raise ValueError("Weave login failed")
+
+        # Then initialize weave client
+        self.weave_client = weave.init(
+            project_name=(f"{self.entity}/{self.project_name}" if self.entity else self.project_name)
+        )
+        self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001")
+        self.calls: dict[str, Any] = {}
+
+    def get_project_url(
+        self,
+    ):
+        try:
+            project_url = f"https://wandb.ai/{self.weave_client._project_id()}"
+            return project_url
+        except Exception as e:
+            logger.debug(f"Weave get run url failed: {str(e)}")
+            raise ValueError(f"Weave get run url failed: {str(e)}")
+
+    def trace(self, trace_info: BaseTraceInfo):
+        logger.debug(f"Trace info: {trace_info}")
+        if isinstance(trace_info, WorkflowTraceInfo):
+            self.workflow_trace(trace_info)
+        if isinstance(trace_info, MessageTraceInfo):
+            self.message_trace(trace_info)
+        if isinstance(trace_info, ModerationTraceInfo):
+            self.moderation_trace(trace_info)
+        if isinstance(trace_info, SuggestedQuestionTraceInfo):
+            self.suggested_question_trace(trace_info)
+        if isinstance(trace_info, DatasetRetrievalTraceInfo):
+            self.dataset_retrieval_trace(trace_info)
+        if isinstance(trace_info, ToolTraceInfo):
+            self.tool_trace(trace_info)
+        if isinstance(trace_info, GenerateNameTraceInfo):
+            self.generate_name_trace(trace_info)
+
+    def workflow_trace(self, trace_info: WorkflowTraceInfo):
+        trace_id = trace_info.message_id or trace_info.workflow_run_id
+        if trace_info.start_time is None:
+            trace_info.start_time = datetime.now()
+
+        if trace_info.message_id:
+            message_attributes = trace_info.metadata
+            message_attributes["workflow_app_log_id"] = trace_info.workflow_app_log_id
+
+            message_attributes["message_id"] = trace_info.message_id
+            message_attributes["workflow_run_id"] = trace_info.workflow_run_id
+            message_attributes["trace_id"] = trace_id
+            message_attributes["start_time"] = trace_info.start_time
+            message_attributes["end_time"] = trace_info.end_time
+            message_attributes["tags"] = ["message", "workflow"]
+
+            message_run = WeaveTraceModel(
+                id=trace_info.message_id,
+                op=str(TraceTaskName.MESSAGE_TRACE.value),
+                inputs=dict(trace_info.workflow_run_inputs),
+                outputs=dict(trace_info.workflow_run_outputs),
+                total_tokens=trace_info.total_tokens,
+                attributes=message_attributes,
+                exception=trace_info.error,
+                file_list=[],
+            )
+            self.start_call(message_run, parent_run_id=trace_info.workflow_run_id)
+            self.finish_call(message_run)
+
+        workflow_attributes = trace_info.metadata
+        workflow_attributes["workflow_run_id"] = trace_info.workflow_run_id
+        workflow_attributes["trace_id"] = trace_id
+        workflow_attributes["start_time"] = trace_info.start_time
+        workflow_attributes["end_time"] = trace_info.end_time
+        workflow_attributes["tags"] = ["workflow"]
+
+        workflow_run = WeaveTraceModel(
+            file_list=trace_info.file_list,
+            total_tokens=trace_info.total_tokens,
+            id=trace_info.workflow_run_id,
+            op=str(TraceTaskName.WORKFLOW_TRACE.value),
+            inputs=dict(trace_info.workflow_run_inputs),
+            outputs=dict(trace_info.workflow_run_outputs),
+            attributes=workflow_attributes,
+            exception=trace_info.error,
+        )
+
+        self.start_call(workflow_run, parent_run_id=trace_info.message_id)
+
+        # through workflow_run_id get all_nodes_execution
+        workflow_nodes_execution_id_records = (
+            db.session.query(WorkflowNodeExecution.id)
+            .filter(WorkflowNodeExecution.workflow_run_id == trace_info.workflow_run_id)
+            .all()
+        )
+
+        for node_execution_id_record in workflow_nodes_execution_id_records:
+            node_execution = (
+                db.session.query(
+                    WorkflowNodeExecution.id,
+                    WorkflowNodeExecution.tenant_id,
+                    WorkflowNodeExecution.app_id,
+                    WorkflowNodeExecution.title,
+                    WorkflowNodeExecution.node_type,
+                    WorkflowNodeExecution.status,
+                    WorkflowNodeExecution.inputs,
+                    WorkflowNodeExecution.outputs,
+                    WorkflowNodeExecution.created_at,
+                    WorkflowNodeExecution.elapsed_time,
+                    WorkflowNodeExecution.process_data,
+                    WorkflowNodeExecution.execution_metadata,
+                )
+                .filter(WorkflowNodeExecution.id == node_execution_id_record.id)
+                .first()
+            )
+
+            if not node_execution:
+                continue
+
+            node_execution_id = node_execution.id
+            tenant_id = node_execution.tenant_id
+            app_id = node_execution.app_id
+            node_name = node_execution.title
+            node_type = node_execution.node_type
+            status = node_execution.status
+            if node_type == "llm":
+                inputs = (
+                    json.loads(node_execution.process_data).get("prompts", {}) if node_execution.process_data else {}
+                )
+            else:
+                inputs = json.loads(node_execution.inputs) if node_execution.inputs else {}
+            outputs = json.loads(node_execution.outputs) if node_execution.outputs else {}
+            created_at = node_execution.created_at or datetime.now()
+            elapsed_time = node_execution.elapsed_time
+            finished_at = created_at + timedelta(seconds=elapsed_time)
+
+            execution_metadata = (
+                json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {}
+            )
+            node_total_tokens = execution_metadata.get("total_tokens", 0)
+            attributes = execution_metadata.copy()
+            attributes.update(
+                {
+                    "workflow_run_id": trace_info.workflow_run_id,
+                    "node_execution_id": node_execution_id,
+                    "tenant_id": tenant_id,
+                    "app_id": app_id,
+                    "app_name": node_name,
+                    "node_type": node_type,
+                    "status": status,
+                }
+            )
+
+            process_data = json.loads(node_execution.process_data) if node_execution.process_data else {}
+            if process_data and process_data.get("model_mode") == "chat":
+                attributes.update(
+                    {
+                        "ls_provider": process_data.get("model_provider", ""),
+                        "ls_model_name": process_data.get("model_name", ""),
+                    }
+                )
+            attributes["tags"] = ["node_execution"]
+            attributes["start_time"] = created_at
+            attributes["end_time"] = finished_at
+            attributes["elapsed_time"] = elapsed_time
+            attributes["workflow_run_id"] = trace_info.workflow_run_id
+            attributes["trace_id"] = trace_id
+            node_run = WeaveTraceModel(
+                total_tokens=node_total_tokens,
+                op=node_type,
+                inputs=inputs,
+                outputs=outputs,
+                file_list=trace_info.file_list,
+                attributes=attributes,
+                id=node_execution_id,
+                exception=None,
+            )
+
+            self.start_call(node_run, parent_run_id=trace_info.workflow_run_id)
+            self.finish_call(node_run)
+
+        self.finish_call(workflow_run)
+
+    def message_trace(self, trace_info: MessageTraceInfo):
+        # get message file data
+        file_list = cast(list[str], trace_info.file_list) or []
+        message_file_data: Optional[MessageFile] = trace_info.message_file_data
+        file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else ""
+        file_list.append(file_url)
+        attributes = trace_info.metadata
+        message_data = trace_info.message_data
+        if message_data is None:
+            return
+        message_id = message_data.id
+
+        user_id = message_data.from_account_id
+        attributes["user_id"] = user_id
+
+        if message_data.from_end_user_id:
+            end_user_data: Optional[EndUser] = (
+                db.session.query(EndUser).filter(EndUser.id == message_data.from_end_user_id).first()
+            )
+            if end_user_data is not None:
+                end_user_id = end_user_data.session_id
+                attributes["end_user_id"] = end_user_id
+
+        attributes["message_id"] = message_id
+        attributes["start_time"] = trace_info.start_time
+        attributes["end_time"] = trace_info.end_time
+        attributes["tags"] = ["message", str(trace_info.conversation_mode)]
+        message_run = WeaveTraceModel(
+            id=message_id,
+            op=str(TraceTaskName.MESSAGE_TRACE.value),
+            input_tokens=trace_info.message_tokens,
+            output_tokens=trace_info.answer_tokens,
+            total_tokens=trace_info.total_tokens,
+            inputs=trace_info.inputs,
+            outputs=trace_info.outputs,
+            exception=trace_info.error,
+            file_list=file_list,
+            attributes=attributes,
+        )
+        self.start_call(message_run)
+
+        # create llm run parented to message run
+        llm_run = WeaveTraceModel(
+            id=str(uuid.uuid4()),
+            input_tokens=trace_info.message_tokens,
+            output_tokens=trace_info.answer_tokens,
+            total_tokens=trace_info.total_tokens,
+            op="llm",
+            inputs=trace_info.inputs,
+            outputs=trace_info.outputs,
+            attributes=attributes,
+            file_list=[],
+            exception=None,
+        )
+        self.start_call(
+            llm_run,
+            parent_run_id=message_id,
+        )
+        self.finish_call(llm_run)
+        self.finish_call(message_run)
+
+    def moderation_trace(self, trace_info: ModerationTraceInfo):
+        if trace_info.message_data is None:
+            return
+
+        attributes = trace_info.metadata
+        attributes["tags"] = ["moderation"]
+        attributes["message_id"] = trace_info.message_id
+        attributes["start_time"] = trace_info.start_time or trace_info.message_data.created_at
+        attributes["end_time"] = trace_info.end_time or trace_info.message_data.updated_at
+
+        moderation_run = WeaveTraceModel(
+            id=str(uuid.uuid4()),
+            op=str(TraceTaskName.MODERATION_TRACE.value),
+            inputs=trace_info.inputs,
+            outputs={
+                "action": trace_info.action,
+                "flagged": trace_info.flagged,
+                "preset_response": trace_info.preset_response,
+                "inputs": trace_info.inputs,
+            },
+            attributes=attributes,
+            exception=getattr(trace_info, "error", None),
+            file_list=[],
+        )
+        self.start_call(moderation_run, parent_run_id=trace_info.message_id)
+        self.finish_call(moderation_run)
+
+    def suggested_question_trace(self, trace_info: SuggestedQuestionTraceInfo):
+        message_data = trace_info.message_data
+        if message_data is None:
+            return
+        attributes = trace_info.metadata
+        attributes["message_id"] = trace_info.message_id
+        attributes["tags"] = ["suggested_question"]
+        attributes["start_time"] = (trace_info.start_time or message_data.created_at,)
+        attributes["end_time"] = (trace_info.end_time or message_data.updated_at,)
+
+        suggested_question_run = WeaveTraceModel(
+            id=str(uuid.uuid4()),
+            op=str(TraceTaskName.SUGGESTED_QUESTION_TRACE.value),
+            inputs=trace_info.inputs,
+            outputs=trace_info.suggested_question,
+            attributes=attributes,
+            exception=trace_info.error,
+            file_list=[],
+        )
+
+        self.start_call(suggested_question_run, parent_run_id=trace_info.message_id)
+        self.finish_call(suggested_question_run)
+
+    def dataset_retrieval_trace(self, trace_info: DatasetRetrievalTraceInfo):
+        if trace_info.message_data is None:
+            return
+        attributes = trace_info.metadata
+        attributes["message_id"] = trace_info.message_id
+        attributes["tags"] = ["dataset_retrieval"]
+        attributes["start_time"] = (trace_info.start_time or trace_info.message_data.created_at,)
+        attributes["end_time"] = (trace_info.end_time or trace_info.message_data.updated_at,)
+
+        dataset_retrieval_run = WeaveTraceModel(
+            id=str(uuid.uuid4()),
+            op=str(TraceTaskName.DATASET_RETRIEVAL_TRACE.value),
+            inputs=trace_info.inputs,
+            outputs={"documents": trace_info.documents},
+            attributes=attributes,
+            exception=getattr(trace_info, "error", None),
+            file_list=[],
+        )
+
+        self.start_call(dataset_retrieval_run, parent_run_id=trace_info.message_id)
+        self.finish_call(dataset_retrieval_run)
+
+    def tool_trace(self, trace_info: ToolTraceInfo):
+        attributes = trace_info.metadata
+        attributes["tags"] = ["tool", trace_info.tool_name]
+        attributes["start_time"] = trace_info.start_time
+        attributes["end_time"] = trace_info.end_time
+
+        tool_run = WeaveTraceModel(
+            id=str(uuid.uuid4()),
+            op=trace_info.tool_name,
+            inputs=trace_info.tool_inputs,
+            outputs=trace_info.tool_outputs,
+            file_list=[cast(str, trace_info.file_url)] if trace_info.file_url else [],
+            attributes=attributes,
+            exception=trace_info.error,
+        )
+        message_id = trace_info.message_id or getattr(trace_info, "conversation_id", None)
+        message_id = message_id or None
+        self.start_call(tool_run, parent_run_id=message_id)
+        self.finish_call(tool_run)
+
+    def generate_name_trace(self, trace_info: GenerateNameTraceInfo):
+        attributes = trace_info.metadata
+        attributes["tags"] = ["generate_name"]
+        attributes["start_time"] = trace_info.start_time
+        attributes["end_time"] = trace_info.end_time
+
+        name_run = WeaveTraceModel(
+            id=str(uuid.uuid4()),
+            op=str(TraceTaskName.GENERATE_NAME_TRACE.value),
+            inputs=trace_info.inputs,
+            outputs=trace_info.outputs,
+            attributes=attributes,
+            exception=getattr(trace_info, "error", None),
+            file_list=[],
+        )
+
+        self.start_call(name_run)
+        self.finish_call(name_run)
+
+    def api_check(self):
+        try:
+            login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True)
+            if not login_status:
+                raise ValueError("Weave login failed")
+            else:
+                print("Weave login successful")
+                return True
+        except Exception as e:
+            logger.debug(f"Weave API check failed: {str(e)}")
+            raise ValueError(f"Weave API check failed: {str(e)}")
+
+    def start_call(self, run_data: WeaveTraceModel, parent_run_id: Optional[str] = None):
+        call = self.weave_client.create_call(op=run_data.op, inputs=run_data.inputs, attributes=run_data.attributes)
+        self.calls[run_data.id] = call
+        if parent_run_id:
+            self.calls[run_data.id].parent_id = parent_run_id
+
+    def finish_call(self, run_data: WeaveTraceModel):
+        call = self.calls.get(run_data.id)
+        if call:
+            self.weave_client.finish_call(call=call, output=run_data.outputs, exception=run_data.exception)
+        else:
+            raise ValueError(f"Call with id {run_data.id} not found")

api/core/plugin/entities/plugin_daemon.py

@@ -1,6 +1,7 @@
+from collections.abc import Mapping
 from datetime import datetime
 from enum import StrEnum
-from typing import Generic, Optional, TypeVar
+from typing import Any, Generic, Optional, TypeVar
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -158,3 +159,11 @@ class PluginInstallTaskStartResponse(BaseModel):
 class PluginUploadResponse(BaseModel):
     unique_identifier: str = Field(description="The unique identifier of the plugin.")
     manifest: PluginDeclaration
+
+
+class PluginOAuthAuthorizationUrlResponse(BaseModel):
+    authorization_url: str = Field(description="The URL of the authorization.")
+
+
+class PluginOAuthCredentialsResponse(BaseModel):
+    credentials: Mapping[str, Any] = Field(description="The credentials of the OAuth.")

api/core/plugin/impl/agent.py

@@ -6,10 +6,10 @@ from core.plugin.entities.plugin import GenericProviderID
 from core.plugin.entities.plugin_daemon import (
     PluginAgentProviderEntity,
 )
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 
 
-class PluginAgentManager(BasePluginManager):
+class PluginAgentClient(BasePluginClient):
     def fetch_agent_strategy_providers(self, tenant_id: str) -> list[PluginAgentProviderEntity]:
         """
         Fetch agent providers for the given tenant.

api/core/plugin/impl/asset.py

@@ -1,7 +1,7 @@
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 
 
-class PluginAssetManager(BasePluginManager):
+class PluginAssetManager(BasePluginClient):
     def fetch_asset(self, tenant_id: str, id: str) -> bytes:
         """
         Fetch an asset by id.

api/core/plugin/impl/base.py

@@ -18,7 +18,7 @@ from core.model_runtime.errors.invoke import (
 )
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.plugin.entities.plugin_daemon import PluginDaemonBasicResponse, PluginDaemonError, PluginDaemonInnerError
-from core.plugin.manager.exc import (
+from core.plugin.impl.exc import (
     PluginDaemonBadRequestError,
     PluginDaemonInternalServerError,
     PluginDaemonNotFoundError,
@@ -37,7 +37,7 @@ T = TypeVar("T", bound=(BaseModel | dict | list | bool | str))
 logger = logging.getLogger(__name__)
 
 
-class BasePluginManager:
+class BasePluginClient:
     def _request(
         self,
         method: str,

api/core/plugin/impl/debugging.py

@@ -1,9 +1,9 @@
 from pydantic import BaseModel
 
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 
 
-class PluginDebuggingManager(BasePluginManager):
+class PluginDebuggingClient(BasePluginClient):
     def get_debugging_key(self, tenant_id: str) -> str:
         """
         Get the debugging key for the given tenant.

api/core/plugin/impl/endpoint.py

@@ -1,8 +1,8 @@
 from core.plugin.entities.endpoint import EndpointEntityWithInstance
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 
 
-class PluginEndpointManager(BasePluginManager):
+class PluginEndpointClient(BasePluginClient):
     def create_endpoint(
         self, tenant_id: str, user_id: str, plugin_unique_identifier: str, name: str, settings: dict
     ) -> bool:

api/core/plugin/impl/model.py

@@ -18,10 +18,10 @@ from core.plugin.entities.plugin_daemon import (
     PluginTextEmbeddingNumTokensResponse,
     PluginVoicesResponse,
 )
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 
 
-class PluginModelManager(BasePluginManager):
+class PluginModelClient(BasePluginClient):
     def fetch_model_providers(self, tenant_id: str) -> Sequence[PluginModelProviderEntity]:
         """
         Fetch model providers for the given tenant.

api/core/plugin/impl/oauth.py

@@ -0,0 +1,98 @@
+from collections.abc import Mapping
+from typing import Any
+
+from werkzeug import Request
+
+from core.plugin.entities.plugin_daemon import PluginOAuthAuthorizationUrlResponse, PluginOAuthCredentialsResponse
+from core.plugin.impl.base import BasePluginClient
+
+
+class OAuthHandler(BasePluginClient):
+    def get_authorization_url(
+        self,
+        tenant_id: str,
+        user_id: str,
+        plugin_id: str,
+        provider: str,
+        system_credentials: Mapping[str, Any],
+    ) -> PluginOAuthAuthorizationUrlResponse:
+        return self._request_with_plugin_daemon_response(
+            "POST",
+            f"plugin/{tenant_id}/dispatch/oauth/get_authorization_url",
+            PluginOAuthAuthorizationUrlResponse,
+            data={
+                "user_id": user_id,
+                "data": {
+                    "provider": provider,
+                    "system_credentials": system_credentials,
+                },
+            },
+            headers={
+                "X-Plugin-ID": plugin_id,
+                "Content-Type": "application/json",
+            },
+        )
+
+    def get_credentials(
+        self,
+        tenant_id: str,
+        user_id: str,
+        plugin_id: str,
+        provider: str,
+        system_credentials: Mapping[str, Any],
+        request: Request,
+    ) -> PluginOAuthCredentialsResponse:
+        """
+        Get credentials from the given request.
+        """
+
+        # encode request to raw http request
+        raw_request_bytes = self._convert_request_to_raw_data(request)
+
+        return self._request_with_plugin_daemon_response(
+            "POST",
+            f"plugin/{tenant_id}/dispatch/oauth/get_credentials",
+            PluginOAuthCredentialsResponse,
+            data={
+                "user_id": user_id,
+                "data": {
+                    "provider": provider,
+                    "system_credentials": system_credentials,
+                    "raw_request_bytes": raw_request_bytes,
+                },
+            },
+            headers={
+                "X-Plugin-ID": plugin_id,
+                "Content-Type": "application/json",
+            },
+        )
+
+    def _convert_request_to_raw_data(self, request: Request) -> bytes:
+        """
+        Convert a Request object to raw HTTP data.
+
+        Args:
+            request: The Request object to convert.
+
+        Returns:
+            The raw HTTP data as bytes.
+        """
+        # Start with the request line
+        method = request.method
+        path = request.path
+        protocol = request.headers.get("HTTP_VERSION", "HTTP/1.1")
+        raw_data = f"{method} {path} {protocol}\r\n".encode()
+
+        # Add headers
+        for header_name, header_value in request.headers.items():
+            raw_data += f"{header_name}: {header_value}\r\n".encode()
+
+        # Add empty line to separate headers from body
+        raw_data += b"\r\n"
+
+        # Add body if exists
+        body = request.get_data(as_text=False)
+        if body:
+            raw_data += body
+
+        return raw_data

api/core/plugin/impl/plugin.py

@@ -10,10 +10,10 @@ from core.plugin.entities.plugin import (
     PluginInstallationSource,
 )
 from core.plugin.entities.plugin_daemon import PluginInstallTask, PluginInstallTaskStartResponse, PluginUploadResponse
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 
 
-class PluginInstallationManager(BasePluginManager):
+class PluginInstaller(BasePluginClient):
     def fetch_plugin_by_identifier(
         self,
         tenant_id: str,

api/core/plugin/impl/tool.py

@@ -5,11 +5,11 @@ from pydantic import BaseModel
 
 from core.plugin.entities.plugin import GenericProviderID, ToolProviderID
 from core.plugin.entities.plugin_daemon import PluginBasicBooleanResponse, PluginToolProviderEntity
-from core.plugin.manager.base import BasePluginManager
+from core.plugin.impl.base import BasePluginClient
 from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
 
 
-class PluginToolManager(BasePluginManager):
+class PluginToolManager(BasePluginClient):
     def fetch_tool_providers(self, tenant_id: str) -> list[PluginToolProviderEntity]:
         """
         Fetch tool providers for the given tenant.

api/core/rag/extractor/watercrawl/provider.py

@@ -20,7 +20,7 @@ class WaterCrawlProvider:
         }
         if options.get("crawl_sub_pages", True):
             spider_options["page_limit"] = options.get("limit", 1)
-            spider_options["max_depth"] = options.get("depth", 1)
+            spider_options["max_depth"] = options.get("max_depth", 1)
             spider_options["include_paths"] = options.get("includes", "").split(",") if options.get("includes") else []
             spider_options["exclude_paths"] = options.get("excludes", "").split(",") if options.get("excludes") else []
 

api/core/tools/plugin_tool/provider.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from core.plugin.manager.tool import PluginToolManager
+from core.plugin.impl.tool import PluginToolManager
 from core.tools.__base.tool_runtime import ToolRuntime
 from core.tools.builtin_tool.provider import BuiltinToolProviderController
 from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin, ToolProviderType

api/core/tools/plugin_tool/tool.py

@@ -1,7 +1,7 @@
 from collections.abc import Generator
 from typing import Any, Optional
 
-from core.plugin.manager.tool import PluginToolManager
+from core.plugin.impl.tool import PluginToolManager
 from core.plugin.utils.converter import convert_parameters_to_plugin_format
 from core.tools.__base.tool import Tool
 from core.tools.__base.tool_runtime import ToolRuntime

api/core/tools/tool_manager.py

@@ -10,7 +10,7 @@ from yarl import URL
 
 import contexts
 from core.plugin.entities.plugin import ToolProviderID
-from core.plugin.manager.tool import PluginToolManager
+from core.plugin.impl.tool import PluginToolManager
 from core.tools.__base.tool_provider import ToolProviderController
 from core.tools.__base.tool_runtime import ToolRuntime
 from core.tools.plugin_tool.provider import PluginToolProviderController

api/core/workflow/nodes/agent/agent_node.py

@@ -7,8 +7,8 @@ from core.agent.plugin_entities import AgentStrategyParameter
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance, ModelManager
 from core.model_runtime.entities.model_entities import AIModelEntity, ModelType
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
-from core.plugin.manager.plugin import PluginInstallationManager
+from core.plugin.impl.plugin import PluginInstaller
 from core.provider_manager import ProviderManager
 from core.tools.entities.tool_entities import ToolParameter, ToolProviderType
 from core.tools.tool_manager import ToolManager
@@ -297,7 +297,7 @@ class AgentNode(ToolNode):
         Get agent strategy icon
         :return:
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         plugins = manager.list_plugins(self.tenant_id)
         try:
             current_plugin = next(

api/core/workflow/nodes/llm/node.py

@@ -1,4 +1,5 @@
 import base64
+import io
 import json
 import logging
 import mimetypes
@@ -79,6 +80,7 @@ from extensions.ext_database import db
 from models.model import Conversation
 from models.provider import Provider, ProviderType
 from models.workflow import WorkflowNodeExecutionStatus
+
 from .entities import (
     LLMNodeChatModelMessage,
     LLMNodeCompletionModelPromptTemplate,
@@ -94,7 +96,6 @@ from .exc import (
     ModelNotExistError,
     NoPromptFoundError,
     TemplateTypeNotSupportError,
-    UnsupportedPromptContentTypeError,
     VariableNotFoundError,
 )
 from .file_downloader import FileDownloader, SSRFProxyFileDownloader
@@ -328,44 +329,31 @@ class LLMNode(BaseNode[LLMNodeData]):
         # For streaming mode
         model = ""
         prompt_messages: list[PromptMessage] = []
-        full_text = ""
+
         usage = LLMUsage.empty_usage()
         finish_reason = None
+        full_text_buffer = io.StringIO()
         for result in invoke_result:
             contents = result.delta.message.content
-            if contents is None:
+            for text_part in self._save_multimodal_output_and_convert_result_to_markdown(contents):
-                continue
+                full_text_buffer.write(text_part)
-
+                yield RunStreamChunkEvent(chunk_content=text_part, from_variable_selector=[self.node_id, "text"])
-            if isinstance(contents, str):
-                yield RunStreamChunkEvent(chunk_content=contents, from_variable_selector=[self.node_id, "text"])
-                full_text += contents
-            elif isinstance(contents, list):
-                for content in contents:
-                    if isinstance(content, TextPromptMessageContent):
-                        text_chunk = content.data
-                    elif isinstance(content, ImagePromptMessageContent):
-                        file = self._save_multimodal_image_output(content)
-                        text_chunk = self._image_file_to_markdown(file)
-                    else:
-                        raise UnsupportedPromptContentTypeError(type_name=str(type(content)))
-                    yield RunStreamChunkEvent(chunk_content=text_chunk, from_variable_selector=[self.node_id, "text"])
-                    full_text += text_chunk
 
             # Update the whole metadata
             if not model and result.model:
                 model = result.model
             if len(prompt_messages) == 0:
+                # TODO(QuantumGhost): it seems that this update has no visable effect.
+                # What's the purpose of the line below?
                 prompt_messages = list(result.prompt_messages)
             if usage.prompt_tokens == 0 and result.delta.usage:
                 usage = result.delta.usage
             if finish_reason is None and result.delta.finish_reason:
                 finish_reason = result.delta.finish_reason
 
-        yield ModelInvokeCompletedEvent(text=full_text, usage=usage, finish_reason=finish_reason)
+        yield ModelInvokeCompletedEvent(text=full_text_buffer.getvalue(), usage=usage, finish_reason=finish_reason)
 
     def _image_file_to_markdown(self, file: "File", /):
-        # TODO(QuantumGhost): Here we have a problem. We must somehow save the file as
-        # a metadata for the workflow, or we cannot regenerate the link.
         text_chunk = f"![]({file.generate_url()})"
         return text_chunk
 
@@ -1026,28 +1014,12 @@ class LLMNode(BaseNode[LLMNodeData]):
         return prompt_messages
 
     def _handle_blocking_result(self, *, invoke_result: LLMResult) -> ModelInvokeCompletedEvent:
-        contents = invoke_result.message.content
+        buffer = io.StringIO()
-        if contents is None:
+        for text_part in self._save_multimodal_output_and_convert_result_to_markdown(invoke_result.message.content):
-            message_text = ""
+            buffer.write(text_part)
-        elif isinstance(contents, str):
-            message_text = contents
-        elif isinstance(contents, list):
-            # TODO: support multi modal content
-            message_text = ""
-            for item in contents:
-                if isinstance(item, TextPromptMessageContent):
-                    message_text += item.data
-                elif isinstance(item, ImagePromptMessageContent):
-                    file = self._save_multimodal_image_output(item)
-                    self._file_outputs.append(file)
-                    message_text += self._image_file_to_markdown(file)
-                else:
-                    message_text += str(item)
-        else:
-            message_text = str(contents)
 
         return ModelInvokeCompletedEvent(
-            text=message_text,
+            text=buffer.getvalue(),
             usage=invoke_result.usage,
             finish_reason=None,
         )
@@ -1258,6 +1230,38 @@ class LLMNode(BaseNode[LLMNodeData]):
             else SupportStructuredOutputStatus.UNSUPPORTED
         )
 
+    def _save_multimodal_output_and_convert_result_to_markdown(
+        self,
+        contents: str | list[PromptMessageContentUnionTypes] | None,
+    ) -> Generator[str, None, None]:
+        """Convert intermediate prompt messages to strings, and yield it to the caller.
+
+        If there are contents other than `TextPromptMessageContent`, it will be saved and
+        correspond markdown content will be yielded to the caller.
+        """
+
+        # NOTE(QuantumGhost): This function should yield to the caller as soon as there is new
+        # content or new content part available. It should avoid intermediate buffering.
+        # It also should avoid yield empty string. (Yield from an empty list instead.)
+        if contents is None:
+            yield from []
+        if isinstance(contents, str):
+            yield contents
+        elif isinstance(contents, list):
+            for item in contents:
+                if isinstance(item, TextPromptMessageContent):
+                    yield item.data
+                elif isinstance(item, ImagePromptMessageContent):
+                    file = self._save_multimodal_image_output(item)
+                    self._file_outputs.append(file)
+                    yield self._image_file_to_markdown(file)
+                else:
+                    logger.warning("unknown item type encountered, type=%s", type(item))
+                    yield str(item)
+        else:
+            logger.warning("unknown contents type encountered, type=%s", type(contents))
+            yield str(contents)
+
 
 def _combine_message_content_with_role(
     *, contents: Optional[str | list[PromptMessageContentUnionTypes]] = None, role: PromptMessageRole

api/core/workflow/nodes/tool/tool_node.py

@@ -6,8 +6,8 @@ from sqlalchemy.orm import Session
 
 from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
 from core.file import File, FileTransferMethod
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
-from core.plugin.manager.plugin import PluginInstallationManager
+from core.plugin.impl.plugin import PluginInstaller
 from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
 from core.tools.errors import ToolInvokeError
 from core.tools.tool_engine import ToolEngine
@@ -307,7 +307,7 @@ class ToolNode(BaseNode[ToolNodeData]):
                     icon = tool_info.get("icon", "")
                     dict_metadata = dict(message.message.metadata)
                     if dict_metadata.get("provider"):
-                        manager = PluginInstallationManager()
+                        manager = PluginInstaller()
                         plugins = manager.list_plugins(self.tenant_id)
                         try:
                             current_plugin = next(

api/extensions/ext_commands.py

@@ -5,6 +5,7 @@ def init_app(app: DifyApp):
     from commands import (
         add_qdrant_index,
         clear_free_plan_tenant_expired_logs,
+        clear_orphaned_file_records,
         convert_to_agent_apps,
         create_tenant,
         extract_plugins,
@@ -13,6 +14,7 @@ def init_app(app: DifyApp):
         install_plugins,
         migrate_data_for_plugin,
         old_metadata_migration,
+        remove_orphaned_files_on_storage,
         reset_email,
         reset_encrypt_key_pair,
         reset_password,
@@ -36,6 +38,8 @@ def init_app(app: DifyApp):
         install_plugins,
         old_metadata_migration,
         clear_free_plan_tenant_expired_logs,
+        clear_orphaned_file_records,
+        remove_orphaned_files_on_storage,
     ]
     for cmd in cmds_to_register:
         app.cli.add_command(cmd)

api/extensions/ext_otel.py

@@ -36,6 +36,31 @@ from configs import dify_config
 from dify_app import DifyApp
 
 
+class ExceptionLoggingHandler(logging.Handler):
+    """Custom logging handler that creates spans for logging.exception() calls"""
+
+    def emit(self, record):
+        try:
+            if record.exc_info:
+                tracer = get_tracer_provider().get_tracer("dify.exception.logging")
+                with tracer.start_as_current_span(
+                    "log.exception",
+                    attributes={
+                        "log.level": record.levelname,
+                        "log.message": record.getMessage(),
+                        "log.logger": record.name,
+                        "log.file.path": record.pathname,
+                        "log.file.line": record.lineno,
+                    },
+                ) as span:
+                    span.set_status(StatusCode.ERROR)
+                    span.record_exception(record.exc_info[1])
+                    span.set_attribute("exception.type", record.exc_info[0].__name__)
+                    span.set_attribute("exception.message", str(record.exc_info[1]))
+        except Exception:
+            pass
+
+
 @user_logged_in.connect
 @user_loaded_from_request.connect
 def on_user_loaded(_sender, user):
@@ -103,6 +128,7 @@ def init_app(app: DifyApp):
         if not is_celery_worker():
             init_flask_instrumentor(app)
             CeleryInstrumentor(tracer_provider=get_tracer_provider(), meter_provider=get_meter_provider()).instrument()
+        instrument_exception_logging()
         init_sqlalchemy_instrumentor(app)
         atexit.register(shutdown_tracer)
 
@@ -111,6 +137,11 @@ def is_celery_worker():
     return "celery" in sys.argv[0].lower()
 
 
+def instrument_exception_logging():
+    exception_handler = ExceptionLoggingHandler()
+    logging.getLogger().addHandler(exception_handler)
+
+
 def init_flask_instrumentor(app: DifyApp):
     meter = get_meter("http_metrics", version=dify_config.CURRENT_VERSION)
     _http_response_counter = meter.create_counter(

api/extensions/ext_storage.py

@@ -102,6 +102,9 @@ class Storage:
     def delete(self, filename):
         return self.storage_runner.delete(filename)
 
+    def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+        return self.storage_runner.scan(path, files=files, directories=directories)
+
 
 storage = Storage()
 

api/extensions/storage/base_storage.py

@@ -30,3 +30,11 @@ class BaseStorage(ABC):
     @abstractmethod
     def delete(self, filename):
         raise NotImplementedError
+
+    def scan(self, path, files=True, directories=False) -> list[str]:
+        """
+        Scan files and directories in the given path.
+        This method is implemented only in some storage backends.
+        If a storage backend doesn't support scanning, it will raise NotImplementedError.
+        """
+        raise NotImplementedError("This storage backend doesn't support scanning")

api/extensions/storage/opendal_storage.py

@@ -80,3 +80,20 @@ class OpenDALStorage(BaseStorage):
             logger.debug(f"file {filename} deleted")
             return
         logger.debug(f"file {filename} not found, skip delete")
+
+    def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+        if not self.exists(path):
+            raise FileNotFoundError("Path not found")
+
+        all_files = self.op.scan(path=path)
+        if files and directories:
+            logger.debug(f"files and directories on {path} scanned")
+            return [f.path for f in all_files]
+        if files:
+            logger.debug(f"files on {path} scanned")
+            return [f.path for f in all_files if not f.path.endswith("/")]
+        elif directories:
+            logger.debug(f"directories on {path} scanned")
+            return [f.path for f in all_files if f.path.endswith("/")]
+        else:
+            raise ValueError("At least one of files or directories must be True")

api/factories/agent_factory.py

@@ -1,12 +1,12 @@
 from core.agent.strategy.plugin import PluginAgentStrategy
-from core.plugin.manager.agent import PluginAgentManager
+from core.plugin.impl.agent import PluginAgentClient
 
 
 def get_plugin_agent_strategy(
     tenant_id: str, agent_strategy_provider_name: str, agent_strategy_name: str
 ) -> PluginAgentStrategy:
     # TODO: use contexts to cache the agent provider
-    manager = PluginAgentManager()
+    manager = PluginAgentClient()
     agent_provider = manager.fetch_agent_strategy_provider(tenant_id, agent_strategy_provider_name)
     for agent_strategy in agent_provider.declaration.strategies:
         if agent_strategy.identity.name == agent_strategy_name:

api/pyproject.toml

@@ -82,6 +82,7 @@ dependencies = [
     "transformers~=4.35.0",
     "unstructured[docx,epub,md,ppt,pptx]~=0.16.1",
     "validators==0.21.0",
+    "weave~=0.51.34",
     "yarl~=1.18.3",
 ]
 # Before adding new dependency, consider place it in

api/services/agent_service.py

@@ -6,8 +6,8 @@ from flask_login import current_user  # type: ignore
 
 import contexts
 from core.app.app_config.easy_ui_based_app.agent.manager import AgentConfigManager
-from core.plugin.manager.agent import PluginAgentManager
+from core.plugin.impl.agent import PluginAgentClient
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
 from core.tools.tool_manager import ToolManager
 from extensions.ext_database import db
 from models.account import Account
@@ -161,7 +161,7 @@ class AgentService:
         """
         List agent providers
         """
-        manager = PluginAgentManager()
+        manager = PluginAgentClient()
         return manager.fetch_agent_strategy_providers(tenant_id)
 
     @classmethod
@@ -169,7 +169,7 @@ class AgentService:
         """
         Get agent provider
         """
-        manager = PluginAgentManager()
+        manager = PluginAgentClient()
         try:
             return manager.fetch_agent_strategy_provider(tenant_id, provider_name)
         except PluginDaemonClientSideError as e:

api/services/ops_service.py

@@ -67,7 +67,14 @@ class OpsService:
                 new_decrypt_tracing_config.update({"project_url": project_url})
             except Exception:
                 new_decrypt_tracing_config.update({"project_url": "https://www.comet.com/opik/"})
-
+        if tracing_provider == "weave" and (
+            "project_url" not in decrypt_tracing_config or not decrypt_tracing_config.get("project_url")
+        ):
+            try:
+                project_url = OpsTraceManager.get_trace_config_project_url(decrypt_tracing_config, tracing_provider)
+                new_decrypt_tracing_config.update({"project_url": project_url})
+            except Exception:
+                new_decrypt_tracing_config.update({"project_url": "https://wandb.ai/"})
         trace_config_data.tracing_config = new_decrypt_tracing_config
         return trace_config_data.to_dict()
 

api/services/plugin/dependencies_analysis.py

@@ -1,7 +1,7 @@
 from configs import dify_config
 from core.helper import marketplace
 from core.plugin.entities.plugin import ModelProviderID, PluginDependency, PluginInstallationSource, ToolProviderID
-from core.plugin.manager.plugin import PluginInstallationManager
+from core.plugin.impl.plugin import PluginInstaller
 
 
 class DependenciesAnalysisService:
@@ -38,7 +38,7 @@ class DependenciesAnalysisService:
         for dependency in dependencies:
             required_plugin_unique_identifiers.append(dependency.value.plugin_unique_identifier)
 
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
 
         # get leaked dependencies
         missing_plugins = manager.fetch_missing_dependencies(tenant_id, required_plugin_unique_identifiers)
@@ -64,7 +64,7 @@ class DependenciesAnalysisService:
         Generate dependencies through the list of plugin ids
         """
         dependencies = list(set(dependencies))
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         plugins = manager.fetch_plugin_installation_by_ids(tenant_id, dependencies)
         result = []
         for plugin in plugins:

api/services/plugin/endpoint_service.py

@@ -1,10 +1,10 @@
-from core.plugin.manager.endpoint import PluginEndpointManager
+from core.plugin.impl.endpoint import PluginEndpointClient
 
 
 class EndpointService:
     @classmethod
     def create_endpoint(cls, tenant_id: str, user_id: str, plugin_unique_identifier: str, name: str, settings: dict):
-        return PluginEndpointManager().create_endpoint(
+        return PluginEndpointClient().create_endpoint(
             tenant_id=tenant_id,
             user_id=user_id,
             plugin_unique_identifier=plugin_unique_identifier,
@@ -14,7 +14,7 @@ class EndpointService:
 
     @classmethod
     def list_endpoints(cls, tenant_id: str, user_id: str, page: int, page_size: int):
-        return PluginEndpointManager().list_endpoints(
+        return PluginEndpointClient().list_endpoints(
             tenant_id=tenant_id,
             user_id=user_id,
             page=page,
@@ -23,7 +23,7 @@ class EndpointService:
 
     @classmethod
     def list_endpoints_for_single_plugin(cls, tenant_id: str, user_id: str, plugin_id: str, page: int, page_size: int):
-        return PluginEndpointManager().list_endpoints_for_single_plugin(
+        return PluginEndpointClient().list_endpoints_for_single_plugin(
             tenant_id=tenant_id,
             user_id=user_id,
             plugin_id=plugin_id,
@@ -33,7 +33,7 @@ class EndpointService:
 
     @classmethod
     def update_endpoint(cls, tenant_id: str, user_id: str, endpoint_id: str, name: str, settings: dict):
-        return PluginEndpointManager().update_endpoint(
+        return PluginEndpointClient().update_endpoint(
             tenant_id=tenant_id,
             user_id=user_id,
             endpoint_id=endpoint_id,
@@ -43,7 +43,7 @@ class EndpointService:
 
     @classmethod
     def delete_endpoint(cls, tenant_id: str, user_id: str, endpoint_id: str):
-        return PluginEndpointManager().delete_endpoint(
+        return PluginEndpointClient().delete_endpoint(
             tenant_id=tenant_id,
             user_id=user_id,
             endpoint_id=endpoint_id,
@@ -51,7 +51,7 @@ class EndpointService:
 
     @classmethod
     def enable_endpoint(cls, tenant_id: str, user_id: str, endpoint_id: str):
-        return PluginEndpointManager().enable_endpoint(
+        return PluginEndpointClient().enable_endpoint(
             tenant_id=tenant_id,
             user_id=user_id,
             endpoint_id=endpoint_id,
@@ -59,7 +59,7 @@ class EndpointService:
 
     @classmethod
     def disable_endpoint(cls, tenant_id: str, user_id: str, endpoint_id: str):
-        return PluginEndpointManager().disable_endpoint(
+        return PluginEndpointClient().disable_endpoint(
             tenant_id=tenant_id,
             user_id=user_id,
             endpoint_id=endpoint_id,

api/services/plugin/oauth_service.py

@@ -0,0 +1,7 @@
+from core.plugin.impl.base import BasePluginClient
+
+
+class OAuthService(BasePluginClient):
+    @classmethod
+    def get_authorization_url(cls, tenant_id: str, user_id: str, provider_name: str) -> str:
+        return "1234567890"

api/services/plugin/plugin_migration.py

@@ -17,7 +17,7 @@ from core.agent.entities import AgentToolEntity
 from core.helper import marketplace
 from core.plugin.entities.plugin import ModelProviderID, PluginInstallationSource, ToolProviderID
 from core.plugin.entities.plugin_daemon import PluginInstallTaskStatus
-from core.plugin.manager.plugin import PluginInstallationManager
+from core.plugin.impl.plugin import PluginInstaller
 from core.tools.entities.tool_entities import ToolProviderType
 from models.account import Tenant
 from models.engine import db
@@ -331,7 +331,7 @@ class PluginMigration:
         """
         Install plugins.
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
 
         plugins = cls.extract_unique_plugins(extracted_plugins)
         not_installed = []
@@ -426,7 +426,7 @@ class PluginMigration:
         """
         Install plugins for a tenant.
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
 
         # download all the plugins and upload
         thread_pool = ThreadPoolExecutor(max_workers=10)

api/services/plugin/plugin_service.py

@@ -18,9 +18,9 @@ from core.plugin.entities.plugin import (
     PluginInstallationSource,
 )
 from core.plugin.entities.plugin_daemon import PluginInstallTask, PluginUploadResponse
-from core.plugin.manager.asset import PluginAssetManager
+from core.plugin.impl.asset import PluginAssetManager
-from core.plugin.manager.debugging import PluginDebuggingManager
+from core.plugin.impl.debugging import PluginDebuggingClient
-from core.plugin.manager.plugin import PluginInstallationManager
+from core.plugin.impl.plugin import PluginInstaller
 from extensions.ext_redis import redis_client
 
 logger = logging.getLogger(__name__)
@@ -91,7 +91,7 @@ class PluginService:
         """
         get the debugging key of the tenant
         """
-        manager = PluginDebuggingManager()
+        manager = PluginDebuggingClient()
         return manager.get_debugging_key(tenant_id)
 
     @staticmethod
@@ -106,7 +106,7 @@ class PluginService:
         """
         list all plugins of the tenant
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         plugins = manager.list_plugins(tenant_id)
         return plugins
 
@@ -115,7 +115,7 @@ class PluginService:
         """
         List plugin installations from ids
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.fetch_plugin_installation_by_ids(tenant_id, ids)
 
     @staticmethod
@@ -133,7 +133,7 @@ class PluginService:
         """
         check if the plugin unique identifier is already installed by other tenant
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.fetch_plugin_by_identifier(tenant_id, plugin_unique_identifier)
 
     @staticmethod
@@ -141,7 +141,7 @@ class PluginService:
         """
         Fetch plugin manifest
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.fetch_plugin_manifest(tenant_id, plugin_unique_identifier)
 
     @staticmethod
@@ -149,12 +149,12 @@ class PluginService:
         """
         Fetch plugin installation tasks
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.fetch_plugin_installation_tasks(tenant_id, page, page_size)
 
     @staticmethod
     def fetch_install_task(tenant_id: str, task_id: str) -> PluginInstallTask:
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.fetch_plugin_installation_task(tenant_id, task_id)
 
     @staticmethod
@@ -162,7 +162,7 @@ class PluginService:
         """
         Delete a plugin installation task
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.delete_plugin_installation_task(tenant_id, task_id)
 
     @staticmethod
@@ -172,7 +172,7 @@ class PluginService:
         """
         Delete all plugin installation task items
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.delete_all_plugin_installation_task_items(tenant_id)
 
     @staticmethod
@@ -180,7 +180,7 @@ class PluginService:
         """
         Delete a plugin installation task item
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.delete_plugin_installation_task_item(tenant_id, task_id, identifier)
 
     @staticmethod
@@ -190,11 +190,14 @@ class PluginService:
         """
         Upgrade plugin with marketplace
         """
+        if not dify_config.MARKETPLACE_ENABLED:
+            raise ValueError("marketplace is not enabled")
+
         if original_plugin_unique_identifier == new_plugin_unique_identifier:
             raise ValueError("you should not upgrade plugin with the same plugin")
 
         # check if plugin pkg is already downloaded
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
 
         try:
             manager.fetch_plugin_manifest(tenant_id, new_plugin_unique_identifier)
@@ -227,7 +230,7 @@ class PluginService:
         """
         Upgrade plugin with github
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.upgrade_plugin(
             tenant_id,
             original_plugin_unique_identifier,
@@ -247,7 +250,7 @@ class PluginService:
 
         returns: plugin_unique_identifier
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.upload_pkg(tenant_id, pkg, verify_signature)
 
     @staticmethod
@@ -262,7 +265,7 @@ class PluginService:
             f"https://github.com/{repo}/releases/download/{version}/{package}", dify_config.PLUGIN_MAX_PACKAGE_SIZE
         )
 
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.upload_pkg(
             tenant_id,
             pkg,
@@ -276,12 +279,12 @@ class PluginService:
         """
         Upload a plugin bundle and return the dependencies.
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.upload_bundle(tenant_id, bundle, verify_signature)
 
     @staticmethod
     def install_from_local_pkg(tenant_id: str, plugin_unique_identifiers: Sequence[str]):
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.install_from_identifiers(
             tenant_id,
             plugin_unique_identifiers,
@@ -295,7 +298,7 @@ class PluginService:
         Install plugin from github release package files,
         returns plugin_unique_identifier
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.install_from_identifiers(
             tenant_id,
             [plugin_unique_identifier],
@@ -316,7 +319,10 @@ class PluginService:
         """
         Fetch marketplace package
         """
-        manager = PluginInstallationManager()
+        if not dify_config.MARKETPLACE_ENABLED:
+            raise ValueError("marketplace is not enabled")
+
+        manager = PluginInstaller()
         try:
             declaration = manager.fetch_plugin_manifest(tenant_id, plugin_unique_identifier)
         except Exception:
@@ -333,7 +339,10 @@ class PluginService:
         Install plugin from marketplace package files,
         returns installation task id
         """
-        manager = PluginInstallationManager()
+        if not dify_config.MARKETPLACE_ENABLED:
+            raise ValueError("marketplace is not enabled")
+
+        manager = PluginInstaller()
 
         # check if already downloaded
         for plugin_unique_identifier in plugin_unique_identifiers:
@@ -359,7 +368,7 @@ class PluginService:
 
     @staticmethod
     def uninstall(tenant_id: str, plugin_installation_id: str) -> bool:
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.uninstall(tenant_id, plugin_installation_id)
 
     @staticmethod
@@ -367,5 +376,5 @@ class PluginService:
         """
         Check if the tools exist
         """
-        manager = PluginInstallationManager()
+        manager = PluginInstaller()
         return manager.check_tools_existence(tenant_id, provider_ids)

api/services/tools/builtin_tools_manage_service.py

@@ -8,7 +8,7 @@ from configs import dify_config
 from core.helper.position_helper import is_filtered
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.plugin.entities.plugin import GenericProviderID, ToolProviderID
-from core.plugin.manager.exc import PluginDaemonClientSideError
+from core.plugin.impl.exc import PluginDaemonClientSideError
 from core.tools.builtin_tool.providers._positions import BuiltinToolProviderSort
 from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity
 from core.tools.errors import ToolNotFoundError, ToolProviderCredentialValidationError, ToolProviderNotFoundError

api/tasks/ops_trace_task.py

@@ -44,7 +44,10 @@ def process_trace_tasks(file_info):
                     trace_info = trace_type(**trace_info)
                 trace_instance.trace(trace_info)
         logging.info(f"Processing trace tasks success, app_id: {app_id}")
-    except Exception:
+    except Exception as e:
+        logging.info(
+            f"error:\n\n\n{e}\n\n\n\n",
+        )
         failed_key = f"{OPS_TRACE_FAILED_KEY}_{app_id}"
         redis_client.incr(failed_key)
         logging.info(f"Processing trace tasks failed, app_id: {app_id}")

api/tests/integration_tests/model_runtime/__mock/plugin_daemon.py

@@ -6,7 +6,7 @@ import pytest
 # import monkeypatch
 from _pytest.monkeypatch import MonkeyPatch
 
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 from tests.integration_tests.model_runtime.__mock.plugin_model import MockModelClass
 
 
@@ -23,9 +23,9 @@ def mock_plugin_daemon(
     def unpatch() -> None:
         monkeypatch.undo()
 
-    monkeypatch.setattr(PluginModelManager, "invoke_llm", MockModelClass.invoke_llm)
+    monkeypatch.setattr(PluginModelClient, "invoke_llm", MockModelClass.invoke_llm)
-    monkeypatch.setattr(PluginModelManager, "fetch_model_providers", MockModelClass.fetch_model_providers)
+    monkeypatch.setattr(PluginModelClient, "fetch_model_providers", MockModelClass.fetch_model_providers)
-    monkeypatch.setattr(PluginModelManager, "get_model_schema", MockModelClass.get_model_schema)
+    monkeypatch.setattr(PluginModelClient, "get_model_schema", MockModelClass.get_model_schema)
 
     return unpatch
 

api/tests/integration_tests/model_runtime/__mock/plugin_model.py

@@ -19,10 +19,10 @@ from core.model_runtime.entities.model_entities import (
 )
 from core.model_runtime.entities.provider_entities import ConfigurateMethod, ProviderEntity
 from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
-from core.plugin.manager.model import PluginModelManager
+from core.plugin.impl.model import PluginModelClient
 
 
-class MockModelClass(PluginModelManager):
+class MockModelClass(PluginModelClient):
     def fetch_model_providers(self, tenant_id: str) -> Sequence[PluginModelProviderEntity]:
         """
         Fetch model providers for the given tenant.
@@ -232,7 +232,7 @@ class MockModelClass(PluginModelManager):
                 )
 
     def invoke_llm(
-        self: PluginModelManager,
+        self: PluginModelClient,
         *,
         tenant_id: str,
         user_id: str,

api/tests/integration_tests/plugin/tools/test_fetch_all_tools.py

@@ -1,4 +1,4 @@
-from core.plugin.manager.tool import PluginToolManager
+from core.plugin.impl.tool import PluginToolManager
 from tests.integration_tests.plugin.__mock.http import setup_http_mock
 
 

api/tests/unit_tests/utils/http_parser/test_oauth_convert_request_to_raw_data.py

@@ -0,0 +1,20 @@
+from werkzeug import Request
+from werkzeug.datastructures import Headers
+from werkzeug.test import EnvironBuilder
+
+from core.plugin.impl.oauth import OAuthHandler
+
+
+def test_oauth_convert_request_to_raw_data():
+    oauth_handler = OAuthHandler()
+    builder = EnvironBuilder(
+        method="GET",
+        path="/test",
+        headers=Headers({"Content-Type": "application/json"}),
+    )
+    request = Request(builder.get_environ())
+    raw_request_bytes = oauth_handler._convert_request_to_raw_data(request)
+
+    assert b"GET /test HTTP/1.1" in raw_request_bytes
+    assert b"Content-Type: application/json" in raw_request_bytes
+    assert b"\r\n\r\n" in raw_request_bytes

api/uv.lock

@@ -1,12 +1,19 @@
 version = 1
+revision = 1
 requires-python = ">=3.11, <3.13"
 resolution-markers = [
-    "python_full_version >= '3.12.4' and platform_python_implementation != 'PyPy'",
+    "python_full_version >= '3.12.4' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'",
-    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_python_implementation != 'PyPy'",
+    "python_full_version >= '3.12.4' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'",
-    "python_full_version >= '3.12.4' and platform_python_implementation == 'PyPy'",
+    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'",
-    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_python_implementation == 'PyPy'",
+    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'",
-    "python_full_version < '3.12' and platform_python_implementation != 'PyPy'",
+    "python_full_version >= '3.12.4' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'",
-    "python_full_version < '3.12' and platform_python_implementation == 'PyPy'",
+    "python_full_version >= '3.12.4' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'",
+    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'",
+    "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'",
+    "python_full_version < '3.12' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'",
+    "python_full_version < '3.12' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'",
+    "python_full_version < '3.12' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'",
+    "python_full_version < '3.12' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'",
 ]
 
 [[package]]
@@ -627,7 +634,7 @@ name = "build"
 version = "1.2.2.post1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "colorama", marker = "os_name == 'nt'" },
+    { name = "colorama", marker = "os_name == 'nt' and sys_platform != 'linux'" },
     { name = "packaging" },
     { name = "pyproject-hooks" },
 ]
@@ -1227,6 +1234,7 @@ dependencies = [
     { name = "transformers" },
     { name = "unstructured", extra = ["docx", "epub", "md", "ppt", "pptx"] },
     { name = "validators" },
+    { name = "weave" },
     { name = "yarl" },
 ]
 
@@ -1396,6 +1404,7 @@ requires-dist = [
     { name = "transformers", specifier = "~=4.35.0" },
     { name = "unstructured", extras = ["docx", "epub", "md", "ppt", "pptx"], specifier = "~=0.16.1" },
     { name = "validators", specifier = "==0.21.0" },
+    { name = "weave", specifier = "~=0.51.34" },
     { name = "yarl", specifier = "~=1.18.3" },
 ]
 
@@ -1487,6 +1496,15 @@ vdb = [
     { name = "xinference-client", specifier = "~=1.2.2" },
 ]
 
+[[package]]
+name = "diskcache"
+version = "5.6.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550 },
+]
+
 [[package]]
 name = "distro"
 version = "1.9.0"
@@ -1496,6 +1514,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277 },
 ]
 
+[[package]]
+name = "docker-pycreds"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c5/e6/d1f6c00b7221e2d7c4b470132c931325c8b22c51ca62417e300f5ce16009/docker-pycreds-0.4.0.tar.gz", hash = "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", size = 8754 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl", hash = "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49", size = 8982 },
+]
+
 [[package]]
 name = "docstring-parser"
 version = "0.16"
@@ -1840,6 +1870,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/11/b2/5d20664ef6a077bec9f27f7a7ee761edc64946d0b1e293726a3d074a9a18/gevent-24.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:68bee86b6e1c041a187347ef84cf03a792f0b6c7238378bf6ba4118af11feaae", size = 1541631 },
 ]
 
+[[package]]
+name = "gitdb"
+version = "4.0.12"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "smmap" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794 },
+]
+
+[[package]]
+name = "gitpython"
+version = "3.1.44"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "gitdb" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/89/37df0b71473153574a5cdef8f242de422a0f5d26d7a9e231e6f169b4ad14/gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269", size = 214196 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110", size = 207599 },
+]
+
 [[package]]
 name = "gmpy2"
 version = "2.2.1"
@@ -2087,6 +2141,39 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/47/3a/1a7cac16438f4e5319a0c879416d5e5032c98c3db2874e6e5300b3b475e6/gotrue-2.11.4-py3-none-any.whl", hash = "sha256:712e5018acc00d93cfc6d7bfddc3114eb3c420ab03b945757a8ba38c5fc3caa8", size = 41106 },
 ]
 
+[[package]]
+name = "gql"
+version = "3.5.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "backoff" },
+    { name = "graphql-core" },
+    { name = "yarl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/ef/5298d9d628b6a54b3b810052cb5a935d324fe28d9bfdeb741733d5c2446b/gql-3.5.2.tar.gz", hash = "sha256:07e1325b820c8ba9478e95de27ce9f23250486e7e79113dbb7659a442dc13e74", size = 180502 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ff/71/b028b937992056e721bbf0371e13819fcca0dacde7b3c821f775ed903917/gql-3.5.2-py2.py3-none-any.whl", hash = "sha256:c830ffc38b3997b2a146317b27758305ab3d0da3bde607b49f34e32affb23ba2", size = 74346 },
+]
+
+[package.optional-dependencies]
+aiohttp = [
+    { name = "aiohttp" },
+]
+requests = [
+    { name = "requests" },
+    { name = "requests-toolbelt" },
+]
+
+[[package]]
+name = "graphql-core"
+version = "3.2.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/66/9e/aa527fb09a9d7399d5d7d2aa2da490e4580707652d3b4fc156996ae88a5b/graphql-core-3.2.4.tar.gz", hash = "sha256:acbe2e800980d0e39b4685dd058c2f4042660b89ebca38af83020fd872ff1264", size = 504611 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/33/cc72c4c658c6316f188a60bc4e5a91cd4ceaaa8c3e7e691ac9297e4e72c7/graphql_core-3.2.4-py3-none-any.whl", hash = "sha256:1604f2042edc5f3114f49cac9d77e25863be51b23a54a61a23245cf32f6476f0", size = 203179 },
+]
+
 [[package]]
 name = "greenlet"
 version = "3.1.1"
@@ -3815,6 +3902,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/37/f3/9b18362206b244167c958984b57c7f70a0289bfb59a530dd8af5f699b910/pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5", size = 2375240 },
 ]
 
+[[package]]
+name = "platformdirs"
+version = "4.3.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b6/2d/7d512a3913d60623e7eb945c6d1b4f0bddf1d0b7ada5225274c87e5b53d1/platformdirs-4.3.7.tar.gz", hash = "sha256:eb437d586b6a0986388f0d6f74aa0cde27b48d0e3d66843640bfb6bdcdb6e351", size = 21291 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6d/45/59578566b3275b8fd9157885918fcd0c4d74162928a5310926887b856a51/platformdirs-4.3.7-py3-none-any.whl", hash = "sha256:a03875334331946f13c549dbd8f4bac7a13a50a895a0eb1e8c6a8ace80d40a94", size = 18499 },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.5.0"
@@ -4084,8 +4180,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/af/cd/ed6e429fb0792ce368f66e83246264dd3a7a045b0b1e63043ed22a063ce5/pycryptodome-3.19.1-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:7c9e222d0976f68d0cf6409cfea896676ddc1d98485d601e9508f90f60e2b0a2", size = 2144914 },
     { url = "https://files.pythonhosted.org/packages/f6/23/b064bd4cfbf2cc5f25afcde0e7c880df5b20798172793137ba4b62d82e72/pycryptodome-3.19.1-cp35-abi3-win32.whl", hash = "sha256:4805e053571140cb37cf153b5c72cd324bb1e3e837cbe590a19f69b6cf85fd03", size = 1713105 },
     { url = "https://files.pythonhosted.org/packages/7d/e0/ded1968a5257ab34216a0f8db7433897a2337d59e6d03be113713b346ea2/pycryptodome-3.19.1-cp35-abi3-win_amd64.whl", hash = "sha256:a470237ee71a1efd63f9becebc0ad84b88ec28e6784a2047684b693f458f41b7", size = 1749222 },
-    { url = "https://files.pythonhosted.org/packages/1d/e3/0c9679cd66cf5604b1f070bdf4525a0c01a15187be287d8348b2eafb718e/pycryptodome-3.19.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:ed932eb6c2b1c4391e166e1a562c9d2f020bfff44a0e1b108f67af38b390ea89", size = 1629005 },
-    { url = "https://files.pythonhosted.org/packages/13/75/0d63bf0daafd0580b17202d8a9dd57f28c8487f26146b3e2799b0c5a059c/pycryptodome-3.19.1-pp27-pypy_73-win32.whl", hash = "sha256:81e9d23c0316fc1b45d984a44881b220062336bbdc340aa9218e8d0656587934", size = 1697997 },
 ]
 
 [[package]]
@@ -4939,6 +5033,38 @@ flask = [
     { name = "markupsafe" },
 ]
 
+[[package]]
+name = "setproctitle"
+version = "1.3.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c4/4d/6a840c8d2baa07b57329490e7094f90aac177a1d5226bc919046f1106860/setproctitle-1.3.5.tar.gz", hash = "sha256:1e6eaeaf8a734d428a95d8c104643b39af7d247d604f40a7bebcf3960a853c5e", size = 26737 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ec/4a/9e0243c5df221102fb834a947f5753d9da06ad5f84e36b0e2e93f7865edb/setproctitle-1.3.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1c8dcc250872385f2780a5ea58050b58cbc8b6a7e8444952a5a65c359886c593", size = 17256 },
+    { url = "https://files.pythonhosted.org/packages/c7/a1/76ad2ba6f5bd00609238e3d64eeded4598e742a5f25b5cc1a0efdae5f674/setproctitle-1.3.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ca82fae9eb4800231dd20229f06e8919787135a5581da245b8b05e864f34cc8b", size = 11893 },
+    { url = "https://files.pythonhosted.org/packages/47/3a/75d11fedff5b21ba9a4c5fe3dfa5e596f831d094ef1896713a72e9e38833/setproctitle-1.3.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0424e1d33232322541cb36fb279ea5242203cd6f20de7b4fb2a11973d8e8c2ce", size = 31631 },
+    { url = "https://files.pythonhosted.org/packages/5a/12/58220de5600e0ed2e5562297173187d863db49babb03491ffe9c101299bc/setproctitle-1.3.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fec8340ab543144d04a9d805d80a0aad73fdeb54bea6ff94e70d39a676ea4ec0", size = 32975 },
+    { url = "https://files.pythonhosted.org/packages/fa/c4/fbb308680d83c1c7aa626950308318c6e6381a8273779163a31741f3c752/setproctitle-1.3.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eab441c89f181271ab749077dcc94045a423e51f2fb0b120a1463ef9820a08d0", size = 30126 },
+    { url = "https://files.pythonhosted.org/packages/31/6e/baaf70bd9a881dd8c12cbccdd7ca0ff291024a37044a8245e942e12e7135/setproctitle-1.3.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c371550a2288901a0dcd84192691ebd3197a43c95f3e0b396ed6d1cedf5c6c", size = 31135 },
+    { url = "https://files.pythonhosted.org/packages/a6/dc/d8ab6b1c3d844dc14f596e3cce76604570848f8a67ba6a3812775ed2c015/setproctitle-1.3.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:78288ff5f9c415c56595b2257ad218936dd9fa726b36341b373b31ca958590fe", size = 30874 },
+    { url = "https://files.pythonhosted.org/packages/d4/84/62a359b3aa51228bd88f78b44ebb0256a5b96dd2487881c1e984a59b617d/setproctitle-1.3.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f1f13a25fc46731acab518602bb1149bfd8b5fabedf8290a7c0926d61414769d", size = 29893 },
+    { url = "https://files.pythonhosted.org/packages/e2/d6/b3c52c03ee41e7f006e1a737e0db1c58d1dc28e258b83548e653d0c34f1c/setproctitle-1.3.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1534d6cd3854d035e40bf4c091984cbdd4d555d7579676d406c53c8f187c006f", size = 32293 },
+    { url = "https://files.pythonhosted.org/packages/55/09/c0ba311879d9c05860503a7e2708ace85913b9a816786402a92c664fe930/setproctitle-1.3.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62a01c76708daac78b9688ffb95268c57cb57fa90b543043cda01358912fe2db", size = 30247 },
+    { url = "https://files.pythonhosted.org/packages/9e/43/cc7155461f0b5a48aebdb87d78239ff3a51ebda0905de478d9fa6ab92d9c/setproctitle-1.3.5-cp311-cp311-win32.whl", hash = "sha256:ea07f29735d839eaed985990a0ec42c8aecefe8050da89fec35533d146a7826d", size = 11476 },
+    { url = "https://files.pythonhosted.org/packages/e7/57/6e937ac7aa52db69225f02db2cfdcb66ba1db6fdc65a4ddbdf78e214f72a/setproctitle-1.3.5-cp311-cp311-win_amd64.whl", hash = "sha256:ab3ae11e10d13d514d4a5a15b4f619341142ba3e18da48c40e8614c5a1b5e3c3", size = 12189 },
+    { url = "https://files.pythonhosted.org/packages/2b/19/04755958495de57e4891de50f03e77b3fe9ca6716a86de00faa00ad0ee5a/setproctitle-1.3.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:523424b9be4dea97d95b8a584b183f35c7bab2d0a3d995b01febf5b8a8de90e4", size = 17250 },
+    { url = "https://files.pythonhosted.org/packages/b9/3d/2ca9df5aa49b975296411dcbbe272cdb1c5e514c43b8be7d61751bb71a46/setproctitle-1.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b6ec1d86c1b4d7b5f2bdceadf213310cf24696b82480a2a702194b8a0bfbcb47", size = 11878 },
+    { url = "https://files.pythonhosted.org/packages/36/d6/e90e23b4627e016a4f862d4f892be92c9765dd6bf1e27a48e52cd166d4a3/setproctitle-1.3.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea6c505264275a43e9b2acd2acfc11ac33caf52bc3167c9fced4418a810f6b1c", size = 31940 },
+    { url = "https://files.pythonhosted.org/packages/15/13/167cdd55e00a8e10b36aad79646c3bf3c23fba0c08a9b8db9b74622c1b13/setproctitle-1.3.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b91e68e6685998e6353f296100ecabc313a6cb3e413d66a03d74b988b61f5ff", size = 33370 },
+    { url = "https://files.pythonhosted.org/packages/9b/22/574a110527df133409a75053b7d6ff740993ccf30b8713d042f26840d351/setproctitle-1.3.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc1fda208ae3a2285ad27aeab44c41daf2328abe58fa3270157a739866779199", size = 30628 },
+    { url = "https://files.pythonhosted.org/packages/52/79/78b05c7d792c9167b917acdab1773b1ff73b016560f45d8155be2baa1a82/setproctitle-1.3.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:828727d220e46f048b82289018300a64547b46aaed96bf8810c05fe105426b41", size = 31672 },
+    { url = "https://files.pythonhosted.org/packages/b0/62/4509735be062129694751ac55d5e1fbb6d86fa46a8689b7d5e2c23dae5b0/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:83b016221cf80028b2947be20630faa14e3e72a403e35f0ba29550b4e856767b", size = 31378 },
+    { url = "https://files.pythonhosted.org/packages/72/e7/b394c55934b89f00c2ef7d5e6f18cca5d8dfa26ef628700c4de0c85e3f3d/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6d8a411e752e794d052434139ca4234ffeceeb8d8d8ddc390a9051d7942b2726", size = 30370 },
+    { url = "https://files.pythonhosted.org/packages/13/ee/e1f27bf52d2bec7060bb6311ab0ccede8de98ed5394e3a59e7a14a453fb5/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:50cfbf86b9c63a2c2903f1231f0a58edeb775e651ae1af84eec8430b0571f29b", size = 32875 },
+    { url = "https://files.pythonhosted.org/packages/6e/08/13b561085d2de53b9becfa5578545d99114e9ff2aa3dc151bcaadf80b17e/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f3b5e2eacd572444770026c9dd3ddc7543ce427cdf452d40a408d1e95beefb30", size = 30903 },
+    { url = "https://files.pythonhosted.org/packages/65/f0/6cd06fffff2553be7b0571447d0c0ef8b727ef44cc2d6a33452677a311c8/setproctitle-1.3.5-cp312-cp312-win32.whl", hash = "sha256:cf4e3ded98027de2596c6cc5bbd3302adfb3ca315c848f56516bb0b7e88de1e9", size = 11468 },
+    { url = "https://files.pythonhosted.org/packages/c1/8c/e8a7cb568c4552618838941b332203bfc77ab0f2d67c1cb8f24dee0370ec/setproctitle-1.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:f7a8c01ffd013dda2bed6e7d5cb59fbb609e72f805abf3ee98360f38f7758d9b", size = 12190 },
+]
+
 [[package]]
 name = "setuptools"
 version = "78.1.0"
@@ -4993,6 +5119,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 },
 ]
 
+[[package]]
+name = "smmap"
+version = "5.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303 },
+]
+
 [[package]]
 name = "sniffio"
 version = "1.3.1"
@@ -5876,6 +6011,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 },
 ]
 
+[[package]]
+name = "uuid-utils"
+version = "0.10.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/66/0a/cbdb2eb4845dafeb632d02a18f47b02f87f2ce4f25266f5e3c017976ce89/uuid_utils-0.10.0.tar.gz", hash = "sha256:5db0e1890e8f008657ffe6ded4d9459af724ab114cfe82af1557c87545301539", size = 18828 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/44/54/9d22fa16b19e5d1676eba510f08a9c458d96e2a62ff2c8ebad64251afb18/uuid_utils-0.10.0-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d5a4508feefec62456cd6a41bcdde458d56827d908f226803b886d22a3d5e63", size = 573006 },
+    { url = "https://files.pythonhosted.org/packages/08/8e/f895c6e52aa603e521fbc13b8626ba5dd99b6e2f5a55aa96ba5b232f4c53/uuid_utils-0.10.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dbefc2b9113f9dfe56bdae58301a2b3c53792221410d422826f3d1e3e6555fe7", size = 292543 },
+    { url = "https://files.pythonhosted.org/packages/b6/58/cc4834f377a5e97d6e184408ad96d13042308de56643b6e24afe1f6f34df/uuid_utils-0.10.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffc49c33edf87d1ec8112a9b43e4cf55326877716f929c165a2cc307d31c73d5", size = 323340 },
+    { url = "https://files.pythonhosted.org/packages/37/e3/6aeddf148f6a7dd7759621b000e8c85382ec83f52ae79b60842d1dc3ab6b/uuid_utils-0.10.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0636b6208f69d5a4e629707ad2a89a04dfa8d1023e1999181f6830646ca048a1", size = 329653 },
+    { url = "https://files.pythonhosted.org/packages/0c/00/dd6c2164ace70b7b1671d9129267df331481d7d1e5f9c5e6a564f07953f6/uuid_utils-0.10.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc06452856b724df9dedfc161c3582199547da54aeb81915ec2ed54f92d19b0", size = 365471 },
+    { url = "https://files.pythonhosted.org/packages/b4/e7/0ab8080fcae5462a7b5e555c1cef3d63457baffb97a59b9bc7b005a3ecb1/uuid_utils-0.10.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:263b2589111c61decdd74a762e8f850c9e4386fb78d2cf7cb4dfc537054cda1b", size = 325844 },
+    { url = "https://files.pythonhosted.org/packages/73/39/52d94e9ef75b03f44b39ffc6ac3167e93e74ef4d010a93d25589d9f48540/uuid_utils-0.10.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a558db48b7096de6b4d2d2210d82bba8586a6d55f99106b03bb7d01dc5c5bcd6", size = 344389 },
+    { url = "https://files.pythonhosted.org/packages/7c/29/4824566f62666238290d99c62a58e4ab2a8b9cf2eccf94cebd9b3359131e/uuid_utils-0.10.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:807465067f3c892514230326ac71a79b28a8dfe2c88ecd2d5675fc844f3c76b5", size = 510078 },
+    { url = "https://files.pythonhosted.org/packages/5e/8f/bbcc7130d652462c685f0d3bd26bb214b754215b476340885a4cb50fb89a/uuid_utils-0.10.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:57423d4a2b9d7b916de6dbd75ba85465a28f9578a89a97f7d3e098d9aa4e5d4a", size = 515937 },
+    { url = "https://files.pythonhosted.org/packages/23/f8/34e0c00f5f188604d336713e6a020fcf53b10998e8ab24735a39ab076740/uuid_utils-0.10.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:76d8d660f18ff6b767e319b1b5f927350cd92eafa4831d7ef5b57fdd1d91f974", size = 494111 },
+    { url = "https://files.pythonhosted.org/packages/1a/52/b7f0066cc90a7a9c28d54061ed195cd617fde822e5d6ac3ccc88509c3c44/uuid_utils-0.10.0-cp39-abi3-win32.whl", hash = "sha256:6c11a71489338837db0b902b75e1ba7618d5d29f05fde4f68b3f909177dbc226", size = 173520 },
+    { url = "https://files.pythonhosted.org/packages/8b/15/f04f58094674d333974243fb45d2c740cf4b79186fb707168e57943c84a3/uuid_utils-0.10.0-cp39-abi3-win_amd64.whl", hash = "sha256:11c55ae64f6c0a7a0c741deae8ca2a4eaa11e9c09dbb7bec2099635696034cf7", size = 182965 },
+]
+
 [[package]]
 name = "uuid6"
 version = "2024.7.10"
@@ -5965,6 +6120,36 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/37/da/7ccbe82470dc27e1cfd0466dc637248be906eb8447c28a40c1c74cf617ee/volcengine_compat-1.0.156-py3-none-any.whl", hash = "sha256:4abc149a7601ebad8fa2d28fab50c7945145cf74daecb71bca797b0bdc82c5a5", size = 677272 },
 ]
 
+[[package]]
+name = "wandb"
+version = "0.18.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "docker-pycreds" },
+    { name = "gitpython" },
+    { name = "platformdirs" },
+    { name = "protobuf" },
+    { name = "psutil" },
+    { name = "pyyaml" },
+    { name = "requests" },
+    { name = "sentry-sdk" },
+    { name = "setproctitle" },
+    { name = "setuptools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cc/57/8a61979c40a7a0a5206ef3369ed474326135bf292f172019f35dca97a235/wandb-0.18.3.tar.gz", hash = "sha256:eb2574cea72bc908c6ce1b37edf7a889619e6e06e1b4714eecfe0662ded43c06", size = 8686381 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1c/4a/6fa1d584ecd69cea5b9943ec5cfa36276cbd567efa8709135a7e4ab89cfb/wandb-0.18.3-py3-none-any.whl", hash = "sha256:7da64f7da0ff7572439de10bfd45534e8811e71e78ac2ccc3b818f1c0f3a9aef", size = 5015658 },
+    { url = "https://files.pythonhosted.org/packages/59/8f/deef595ca67833ea5aceb5da5fc10759a5e8f8bce85b17761b1614fa2ba9/wandb-0.18.3-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6674d8a5c40c79065b9c7eb765136756d5ebc9457a5f9abc820a660fb23f8b67", size = 10081571 },
+    { url = "https://files.pythonhosted.org/packages/06/85/b55642d095407369dd7ad1d8ea1e7f410d60fcdb6c29bcc9afb1e5522d51/wandb-0.18.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:741f566e409a2684d3047e4cc25e8e914d78196b901190937b24b6abb8b052e5", size = 10008319 },
+    { url = "https://files.pythonhosted.org/packages/b4/53/5387afaab29876e669973b3bb5bda829e3c10e509caef59f614bf20c0106/wandb-0.18.3-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:8be5e877570b693001c52dcc2089e48e6a4dcbf15f3adf5c9349f95148b59d58", size = 10250633 },
+    { url = "https://files.pythonhosted.org/packages/bd/79/2fa554283afa7259e296313160164947daf52e0d42b04d6ecf9c5af01e15/wandb-0.18.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d788852bd4739fa18de3918f309c3a955b5cef3247fae1c40df3a63af637e1a0", size = 12339454 },
+    { url = "https://files.pythonhosted.org/packages/86/a6/11eaa16c96469b4d6fc0fb3271e70d5bbe2c3a93c15fc677de9a1aa4374a/wandb-0.18.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab81424eb207d78239a8d69c90521a70074fb81e3709055484e43c76fe44dc08", size = 12970950 },
+    { url = "https://files.pythonhosted.org/packages/13/dd/ccaa5a51e2557368300eec9e362b5688151e45a052e33017633baa3011a9/wandb-0.18.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:2c91315b8b62423eae18577d66a4b4bb8e4341a7d5c849cb2963e3b3dff0bf6d", size = 13038220 },
+    { url = "https://files.pythonhosted.org/packages/bc/6f/fabbf2161078556384ef48f3db89182773010cdd14900986004e702b85f5/wandb-0.18.3-py3-none-win32.whl", hash = "sha256:92a647dab783938ec87776a9fae8a13e72e6dad939c53e357cdea9d2570f0ad8", size = 12573298 },
+    { url = "https://files.pythonhosted.org/packages/d8/7b/e94b46d620d26b2e1f486f2746febdcb6579be20f361355b40263ddd8262/wandb-0.18.3-py3-none-win_amd64.whl", hash = "sha256:29cac2cfa3124241fed22cfedc9a52e1500275ee9bbb0b428ce4bf63c4723bf0", size = 12573303 },
+]
+
 [[package]]
 name = "watchfiles"
 version = "1.0.5"
@@ -6011,6 +6196,28 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166 },
 ]
 
+[[package]]
+name = "weave"
+version = "0.51.43"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "diskcache" },
+    { name = "emoji" },
+    { name = "gql", extra = ["aiohttp", "requests"] },
+    { name = "jsonschema" },
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "pydantic" },
+    { name = "rich" },
+    { name = "tenacity" },
+    { name = "uuid-utils" },
+    { name = "wandb" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e1/b4/8fb1e21bc0b0442be9c4c5e4644847596cd75a35a313a5887f1eadda8da2/weave-0.51.43.tar.gz", hash = "sha256:bab4ba6f7ba33f1975e5f6399b7fc4ad6b25c0e2cd22d197bb9358a7b9596b91", size = 368936 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a8/40/1e374d3f1f8389a4228426b5a87aae7428a7eb74dfa633de98d86796eb41/weave-0.51.43-py3-none-any.whl", hash = "sha256:2e9faa0e21bd5a6fea363142891ee4f2e347951b98f0d7082acb0273432cb940", size = 473685 },
+]
+
 [[package]]
 name = "weaviate-client"
 version = "3.21.0"

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx

@@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next'
 import { useBoolean } from 'ahooks'
 import TracingIcon from './tracing-icon'
 import ProviderPanel from './provider-panel'
-import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type'
+import type { LangFuseConfig, LangSmithConfig, OpikConfig, WeaveConfig } from './type'
 import { TracingProvider } from './type'
 import ProviderConfigModal from './provider-config-modal'
 import Indicator from '@/app/components/header/indicator'
@@ -26,7 +26,8 @@ export type PopupProps = {
   langSmithConfig: LangSmithConfig | null
   langFuseConfig: LangFuseConfig | null
   opikConfig: OpikConfig | null
-  onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig | OpikConfig) => void
+  weaveConfig: WeaveConfig | null
+  onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig | OpikConfig | WeaveConfig) => void
   onConfigRemoved: (provider: TracingProvider) => void
 }
 
@@ -40,6 +41,7 @@ const ConfigPopup: FC<PopupProps> = ({
   langSmithConfig,
   langFuseConfig,
   opikConfig,
+  weaveConfig,
   onConfigUpdated,
   onConfigRemoved,
 }) => {
@@ -63,7 +65,7 @@ const ConfigPopup: FC<PopupProps> = ({
     }
   }, [onChooseProvider])
 
-  const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig | OpikConfig) => {
+  const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig | OpikConfig | WeaveConfig) => {
     onConfigUpdated(currentProvider!, payload)
     hideConfigModal()
   }, [currentProvider, hideConfigModal, onConfigUpdated])
@@ -73,8 +75,8 @@ const ConfigPopup: FC<PopupProps> = ({
     hideConfigModal()
   }, [currentProvider, hideConfigModal, onConfigRemoved])
 
-  const providerAllConfigured = langSmithConfig && langFuseConfig && opikConfig
+  const providerAllConfigured = langSmithConfig && langFuseConfig && opikConfig && weaveConfig
-  const providerAllNotConfigured = !langSmithConfig && !langFuseConfig && !opikConfig
+  const providerAllNotConfigured = !langSmithConfig && !langFuseConfig && !opikConfig && !weaveConfig
 
   const switchContent = (
     <Switch
@@ -123,33 +125,51 @@ const ConfigPopup: FC<PopupProps> = ({
     />
   )
 
+  const weavePanel = (
+    <ProviderPanel
+      type={TracingProvider.weave}
+      readOnly={readOnly}
+      config={weaveConfig}
+      hasConfigured={!!weaveConfig}
+      onConfig={handleOnConfig(TracingProvider.weave)}
+      isChosen={chosenProvider === TracingProvider.weave}
+      onChoose={handleOnChoose(TracingProvider.weave)}
+      key="weave-provider-panel"
+    />
+  )
   const configuredProviderPanel = () => {
     const configuredPanels: JSX.Element[] = []
 
-    if (langSmithConfig)
-      configuredPanels.push(langSmithPanel)
-
     if (langFuseConfig)
       configuredPanels.push(langfusePanel)
 
+    if (langSmithConfig)
+      configuredPanels.push(langSmithPanel)
+
     if (opikConfig)
       configuredPanels.push(opikPanel)
 
+    if (weaveConfig)
+      configuredPanels.push(weavePanel)
+
     return configuredPanels
   }
 
   const moreProviderPanel = () => {
     const notConfiguredPanels: JSX.Element[] = []
 
-    if (!langSmithConfig)
-      notConfiguredPanels.push(langSmithPanel)
-
     if (!langFuseConfig)
       notConfiguredPanels.push(langfusePanel)
 
+    if (!langSmithConfig)
+      notConfiguredPanels.push(langSmithPanel)
+
     if (!opikConfig)
       notConfiguredPanels.push(opikPanel)
 
+    if (!weaveConfig)
+      notConfiguredPanels.push(weavePanel)
+
     return notConfiguredPanels
   }
 
@@ -158,7 +178,9 @@ const ConfigPopup: FC<PopupProps> = ({
       return langSmithConfig
     if (currentProvider === TracingProvider.langfuse)
       return langFuseConfig
+    if (currentProvider === TracingProvider.opik)
       return opikConfig
+    return weaveConfig
   }
 
   return (
@@ -199,9 +221,10 @@ const ConfigPopup: FC<PopupProps> = ({
             <>
               <div className='system-xs-medium-uppercase text-text-tertiary'>{t(`${I18N_PREFIX}.configProviderTitle.${providerAllConfigured ? 'configured' : 'notConfigured'}`)}</div>
               <div className='mt-2 space-y-2'>
-                {langSmithPanel}
                 {langfusePanel}
+                {langSmithPanel}
                 {opikPanel}
+                {weavePanel}
               </div>
             </>
           )

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config.ts

@@ -4,4 +4,5 @@ export const docURL = {
   [TracingProvider.langSmith]: 'https://docs.smith.langchain.com/',
   [TracingProvider.langfuse]: 'https://docs.langfuse.com',
   [TracingProvider.opik]: 'https://www.comet.com/docs/opik/tracing/integrations/dify#setup-instructions',
+  [TracingProvider.weave]: 'https://weave-docs.wandb.ai/',
 }

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx

@@ -7,12 +7,12 @@ import {
 import { useTranslation } from 'react-i18next'
 import { usePathname } from 'next/navigation'
 import { useBoolean } from 'ahooks'
-import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type'
+import type { LangFuseConfig, LangSmithConfig, OpikConfig, WeaveConfig } from './type'
 import { TracingProvider } from './type'
 import TracingIcon from './tracing-icon'
 import ConfigButton from './config-button'
 import cn from '@/utils/classnames'
-import { LangfuseIcon, LangsmithIcon, OpikIcon } from '@/app/components/base/icons/src/public/tracing'
+import { LangfuseIcon, LangsmithIcon, OpikIcon, WeaveIcon } from '@/app/components/base/icons/src/public/tracing'
 import Indicator from '@/app/components/header/indicator'
 import { fetchTracingConfig as doFetchTracingConfig, fetchTracingStatus, updateTracingStatus } from '@/service/apps'
 import type { TracingStatus } from '@/models/app'
@@ -82,12 +82,15 @@ const Panel: FC = () => {
         ? LangfuseIcon
         : inUseTracingProvider === TracingProvider.opik
           ? OpikIcon
+          : inUseTracingProvider === TracingProvider.weave
+            ? WeaveIcon
             : LangsmithIcon
 
   const [langSmithConfig, setLangSmithConfig] = useState<LangSmithConfig | null>(null)
   const [langFuseConfig, setLangFuseConfig] = useState<LangFuseConfig | null>(null)
   const [opikConfig, setOpikConfig] = useState<OpikConfig | null>(null)
-  const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig || opikConfig)
+  const [weaveConfig, setWeaveConfig] = useState<WeaveConfig | null>(null)
+  const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig || opikConfig || weaveConfig)
 
   const fetchTracingConfig = async () => {
     const { tracing_config: langSmithConfig, has_not_configured: langSmithHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith })
@@ -99,6 +102,9 @@ const Panel: FC = () => {
     const { tracing_config: opikConfig, has_not_configured: OpikHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.opik })
     if (!OpikHasNotConfig)
       setOpikConfig(opikConfig as OpikConfig)
+    const { tracing_config: weaveConfig, has_not_configured: weaveHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.weave })
+    if (!weaveHasNotConfig)
+      setWeaveConfig(weaveConfig as WeaveConfig)
   }
 
   const handleTracingConfigUpdated = async (provider: TracingProvider) => {
@@ -110,6 +116,8 @@ const Panel: FC = () => {
       setLangFuseConfig(tracing_config as LangFuseConfig)
     else if (provider === TracingProvider.opik)
       setOpikConfig(tracing_config as OpikConfig)
+    else if (provider === TracingProvider.weave)
+      setWeaveConfig(tracing_config as WeaveConfig)
   }
 
   const handleTracingConfigRemoved = (provider: TracingProvider) => {
@@ -119,6 +127,8 @@ const Panel: FC = () => {
       setLangFuseConfig(null)
     else if (provider === TracingProvider.opik)
       setOpikConfig(null)
+    else if (provider === TracingProvider.weave)
+      setWeaveConfig(null)
     if (provider === inUseTracingProvider) {
       handleTracingStatusChange({
         enabled: false,
@@ -178,6 +188,7 @@ const Panel: FC = () => {
                 langSmithConfig={langSmithConfig}
                 langFuseConfig={langFuseConfig}
                 opikConfig={opikConfig}
+                weaveConfig={weaveConfig}
                 onConfigUpdated={handleTracingConfigUpdated}
                 onConfigRemoved={handleTracingConfigRemoved}
                 controlShowPopup={controlShowPopup}
@@ -212,6 +223,7 @@ const Panel: FC = () => {
                 langSmithConfig={langSmithConfig}
                 langFuseConfig={langFuseConfig}
                 opikConfig={opikConfig}
+                weaveConfig={weaveConfig}
                 onConfigUpdated={handleTracingConfigUpdated}
                 onConfigRemoved={handleTracingConfigRemoved}
                 controlShowPopup={controlShowPopup}

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx

@@ -4,7 +4,7 @@ import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useBoolean } from 'ahooks'
 import Field from './field'
-import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type'
+import type { LangFuseConfig, LangSmithConfig, OpikConfig, WeaveConfig } from './type'
 import { TracingProvider } from './type'
 import { docURL } from './config'
 import {
@@ -22,10 +22,10 @@ import Divider from '@/app/components/base/divider'
 type Props = {
   appId: string
   type: TracingProvider
-  payload?: LangSmithConfig | LangFuseConfig | OpikConfig | null
+  payload?: LangSmithConfig | LangFuseConfig | OpikConfig | WeaveConfig | null
   onRemoved: () => void
   onCancel: () => void
-  onSaved: (payload: LangSmithConfig | LangFuseConfig | OpikConfig) => void
+  onSaved: (payload: LangSmithConfig | LangFuseConfig | OpikConfig | WeaveConfig) => void
   onChosen: (provider: TracingProvider) => void
 }
 
@@ -50,6 +50,13 @@ const opikConfigTemplate = {
   workspace: '',
 }
 
+const weaveConfigTemplate = {
+  api_key: '',
+  entity: '',
+  project: '',
+  endpoint: '',
+}
+
 const ProviderConfigModal: FC<Props> = ({
   appId,
   type,
@@ -63,7 +70,7 @@ const ProviderConfigModal: FC<Props> = ({
   const isEdit = !!payload
   const isAdd = !isEdit
   const [isSaving, setIsSaving] = useState(false)
-  const [config, setConfig] = useState<LangSmithConfig | LangFuseConfig | OpikConfig>((() => {
+  const [config, setConfig] = useState<LangSmithConfig | LangFuseConfig | OpikConfig | WeaveConfig>((() => {
     if (isEdit)
       return payload
 
@@ -73,7 +80,10 @@ const ProviderConfigModal: FC<Props> = ({
     else if (type === TracingProvider.langfuse)
       return langFuseConfigTemplate
 
+    else if (type === TracingProvider.opik)
       return opikConfigTemplate
+
+    return weaveConfigTemplate
   })())
   const [isShowRemoveConfirm, {
     setTrue: showRemoveConfirm,
@@ -127,6 +137,14 @@ const ProviderConfigModal: FC<Props> = ({
       // const postData = config as OpikConfig
     }
 
+    if (type === TracingProvider.weave) {
+      const postData = config as WeaveConfig
+      if (!errorMessage && !postData.api_key)
+        errorMessage = t('common.errorMsg.fieldRequired', { field: 'API Key' })
+      if (!errorMessage && !postData.project)
+        errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.project`) })
+    }
+
     return errorMessage
   }, [config, t, type])
   const handleSave = useCallback(async () => {
@@ -176,6 +194,40 @@ const ProviderConfigModal: FC<Props> = ({
                     </div>
 
                     <div className='space-y-4'>
+                      {type === TracingProvider.weave && (
+                        <>
+                          <Field
+                            label='API Key'
+                            labelClassName='!text-sm'
+                            isRequired
+                            value={(config as WeaveConfig).api_key}
+                            onChange={handleConfigChange('api_key')}
+                            placeholder={t(`${I18N_PREFIX}.placeholder`, { key: 'API Key' })!}
+                          />
+                          <Field
+                            label={t(`${I18N_PREFIX}.project`)!}
+                            labelClassName='!text-sm'
+                            isRequired
+                            value={(config as WeaveConfig).project}
+                            onChange={handleConfigChange('project')}
+                            placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.project`) })!}
+                          />
+                          <Field
+                            label='Entity'
+                            labelClassName='!text-sm'
+                            value={(config as WeaveConfig).entity}
+                            onChange={handleConfigChange('entity')}
+                            placeholder={t(`${I18N_PREFIX}.placeholder`, { key: 'Entity' })!}
+                          />
+                          <Field
+                            label='Endpoint'
+                            labelClassName='!text-sm'
+                            value={(config as WeaveConfig).endpoint}
+                            onChange={handleConfigChange('endpoint')}
+                            placeholder={'https://trace.wandb.ai/'}
+                          />
+                        </>
+                      )}
                       {type === TracingProvider.langSmith && (
                         <>
                           <Field
@@ -263,7 +315,6 @@ const ProviderConfigModal: FC<Props> = ({
                           />
                         </>
                       )}
-
                     </div>
                     <div className='my-8 flex h-8 items-center justify-between'>
                       <a

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx

@@ -7,7 +7,7 @@ import {
 import { useTranslation } from 'react-i18next'
 import { TracingProvider } from './type'
 import cn from '@/utils/classnames'
-import { LangfuseIconBig, LangsmithIconBig, OpikIconBig } from '@/app/components/base/icons/src/public/tracing'
+import { LangfuseIconBig, LangsmithIconBig, OpikIconBig, WeaveIconBig } from '@/app/components/base/icons/src/public/tracing'
 import { Eye as View } from '@/app/components/base/icons/src/vender/solid/general'
 
 const I18N_PREFIX = 'app.tracing'
@@ -27,6 +27,7 @@ const getIcon = (type: TracingProvider) => {
     [TracingProvider.langSmith]: LangsmithIconBig,
     [TracingProvider.langfuse]: LangfuseIconBig,
     [TracingProvider.opik]: OpikIconBig,
+    [TracingProvider.weave]: WeaveIconBig,
   })[type]
 }
 

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts

@@ -2,6 +2,7 @@ export enum TracingProvider {
   langSmith = 'langsmith',
   langfuse = 'langfuse',
   opik = 'opik',
+  weave = 'weave',
 }
 
 export type LangSmithConfig = {
@@ -22,3 +23,10 @@ export type OpikConfig = {
   workspace: string
   url: string
 }
+
+export type WeaveConfig = {
+  api_key: string
+  entity: string
+  project: string
+  endpoint: string
+}

web/app/components/app/app-publisher/index.tsx

@@ -231,7 +231,7 @@ const AppPublisher = ({
               >
                 {t('workflow.common.runApp')}
               </SuggestedAction>
-              {appDetail?.mode === 'workflow'
+              {appDetail?.mode === 'workflow' || appDetail?.mode === 'completion'
                 ? (
                   <SuggestedAction
                     disabled={!publishedAt}

web/app/components/app/configuration/dataset-config/settings-modal/index.tsx

@@ -4,7 +4,6 @@ import { useMount } from 'ahooks'
 import { useTranslation } from 'react-i18next'
 import { isEqual } from 'lodash-es'
 import { RiCloseLine } from '@remixicon/react'
-import { BookOpenIcon } from '@heroicons/react/24/outline'
 import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
 import cn from '@/utils/classnames'
 import IndexMethodRadio from '@/app/components/datasets/settings/index-method-radio'
@@ -223,10 +222,6 @@ const SettingsModal: FC<SettingsModalProps> = ({
               className='resize-none'
               placeholder={t('datasetSettings.form.descPlaceholder') || ''}
             />
-            <a className='mt-2 flex h-[18px] items-center px-3 text-xs text-text-tertiary' href="https://docs.dify.ai/features/datasets#how-to-write-a-good-dataset-description" target='_blank' rel='noopener noreferrer'>
-              <BookOpenIcon className='mr-1 h-[18px] w-3' />
-              {t('datasetSettings.form.descWrite')}
-            </a>
           </div>
         </div>
         <div className={rowClass}>

web/app/components/app/create-app-dialog/app-list/index.tsx

@@ -191,14 +191,16 @@ const Apps = ({
       </div>
       <div className='relative flex flex-1 overflow-y-auto'>
         {!searchKeywords && <div className='h-full w-[200px] p-4'>
-          <Sidebar current={currCategory as AppCategories} onClick={(category) => { setCurrCategory(category) }} onCreateFromBlank={onCreateFromBlank} />
+          <Sidebar current={currCategory as AppCategories} categories={categories} onClick={(category) => { setCurrCategory(category) }} onCreateFromBlank={onCreateFromBlank} />
         </div>}
         <div className='h-full flex-1 shrink-0 grow overflow-auto border-l border-divider-burn p-6 pt-2'>
           {searchFilteredList && searchFilteredList.length > 0 && <>
             <div className='pb-1 pt-4'>
               {searchKeywords
                 ? <p className='title-md-semi-bold text-text-tertiary'>{searchFilteredList.length > 1 ? t('app.newApp.foundResults', { count: searchFilteredList.length }) : t('app.newApp.foundResult', { count: searchFilteredList.length })}</p>
-                : <AppCategoryLabel category={currCategory as AppCategories} className='title-md-semi-bold text-text-primary' />}
+                : <div className='flex h-[22px] items-center'>
+                  <AppCategoryLabel category={currCategory as AppCategories} className='title-md-semi-bold text-text-primary' />
+                </div>}
             </div>
             <div
               className={cn(

web/app/components/app/create-app-dialog/app-list/sidebar.tsx

@@ -1,39 +1,29 @@
 'use client'
-import { RiAppsFill, RiChatSmileAiFill, RiExchange2Fill, RiPassPendingFill, RiQuillPenAiFill, RiSpeakAiFill, RiStickyNoteAddLine, RiTerminalBoxFill, RiThumbUpFill } from '@remixicon/react'
+import { RiStickyNoteAddLine, RiThumbUpLine } from '@remixicon/react'
 import { useTranslation } from 'react-i18next'
 import classNames from '@/utils/classnames'
 import Divider from '@/app/components/base/divider'
 
 export enum AppCategories {
   RECOMMENDED = 'Recommended',
-  ASSISTANT = 'Assistant',
-  AGENT = 'Agent',
-  HR = 'HR',
-  PROGRAMMING = 'Programming',
-  WORKFLOW = 'Workflow',
-  WRITING = 'Writing',
 }
 
 type SidebarProps = {
-  current: AppCategories
+  current: AppCategories | string
-  onClick?: (category: AppCategories) => void
+  categories: string[]
+  onClick?: (category: AppCategories | string) => void
   onCreateFromBlank?: () => void
 }
 
-export default function Sidebar({ current, onClick, onCreateFromBlank }: SidebarProps) {
+export default function Sidebar({ current, categories, onClick, onCreateFromBlank }: SidebarProps) {
   const { t } = useTranslation()
   return <div className="flex h-full w-full flex-col">
-    <ul>
+    <ul className='pt-0.5'>
       <CategoryItem category={AppCategories.RECOMMENDED} active={current === AppCategories.RECOMMENDED} onClick={onClick} />
     </ul>
-    <div className='system-xs-medium-uppercase px-3 pb-1 pt-2 text-text-tertiary'>{t('app.newAppFromTemplate.byCategories')}</div>
+    <div className='system-xs-medium-uppercase mb-0.5 mt-3 px-3 pb-1 pt-2 text-text-tertiary'>{t('app.newAppFromTemplate.byCategories')}</div>
     <ul className='flex grow flex-col gap-0.5'>
-      <CategoryItem category={AppCategories.ASSISTANT} active={current === AppCategories.ASSISTANT} onClick={onClick} />
+      {categories.map(category => (<CategoryItem key={category} category={category} active={current === category} onClick={onClick} />))}
-      <CategoryItem category={AppCategories.AGENT} active={current === AppCategories.AGENT} onClick={onClick} />
-      <CategoryItem category={AppCategories.HR} active={current === AppCategories.HR} onClick={onClick} />
-      <CategoryItem category={AppCategories.PROGRAMMING} active={current === AppCategories.PROGRAMMING} onClick={onClick} />
-      <CategoryItem category={AppCategories.WORKFLOW} active={current === AppCategories.WORKFLOW} onClick={onClick} />
-      <CategoryItem category={AppCategories.WRITING} active={current === AppCategories.WRITING} onClick={onClick} />
     </ul>
     <Divider bgStyle='gradient' />
     <div className='flex cursor-pointer items-center gap-1 px-3 py-1 text-text-tertiary' onClick={onCreateFromBlank}>
@@ -45,47 +35,26 @@ export default function Sidebar({ current, onClick, onCreateFromBlank }: Sidebar
 
 type CategoryItemProps = {
   active: boolean
-  category: AppCategories
+  category: AppCategories | string
-  onClick?: (category: AppCategories) => void
+  onClick?: (category: AppCategories | string) => void
 }
 function CategoryItem({ category, active, onClick }: CategoryItemProps) {
   return <li
-    className={classNames('p-1 pl-3 rounded-lg flex items-center gap-2 group cursor-pointer hover:bg-state-base-hover [&.active]:bg-state-base-active', active && 'active')}
+    className={classNames('p-1 pl-3 h-8 rounded-lg flex items-center gap-2 group cursor-pointer hover:bg-state-base-hover [&.active]:bg-state-base-active', active && 'active')}
     onClick={() => { onClick?.(category) }}>
-    <div className='inline-flex h-5 w-5 items-center justify-center rounded-md border border-divider-regular bg-components-icon-bg-midnight-solid group-[.active]:bg-components-icon-bg-blue-solid'>
+    {category === AppCategories.RECOMMENDED && <div className='inline-flex h-5 w-5 items-center justify-center rounded-md'>
-      <AppCategoryIcon category={category} />
+      <RiThumbUpLine className='h-4 w-4 text-components-menu-item-text group-[.active]:text-components-menu-item-text-active' />
-    </div>
+    </div>}
     <AppCategoryLabel category={category}
       className={classNames('system-sm-medium text-components-menu-item-text group-[.active]:text-components-menu-item-text-active group-hover:text-components-menu-item-text-hover', active && 'system-sm-semibold')} />
   </li >
 }
 
 type AppCategoryLabelProps = {
-  category: AppCategories
+  category: AppCategories | string
   className?: string
 }
 export function AppCategoryLabel({ category, className }: AppCategoryLabelProps) {
   const { t } = useTranslation()
-  return <span className={className}>{t(`app.newAppFromTemplate.sidebar.${category}`)}</span>
+  return <span className={className}>{category === AppCategories.RECOMMENDED ? t('app.newAppFromTemplate.sidebar.Recommended') : category}</span>
-}
-
-type AppCategoryIconProps = {
-  category: AppCategories
-}
-function AppCategoryIcon({ category }: AppCategoryIconProps) {
-  if (category === AppCategories.AGENT)
-    return <RiSpeakAiFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  if (category === AppCategories.ASSISTANT)
-    return <RiChatSmileAiFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  if (category === AppCategories.HR)
-    return <RiPassPendingFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  if (category === AppCategories.PROGRAMMING)
-    return <RiTerminalBoxFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  if (category === AppCategories.RECOMMENDED)
-    return <RiThumbUpFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  if (category === AppCategories.WRITING)
-    return <RiQuillPenAiFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  if (category === AppCategories.WORKFLOW)
-    return <RiExchange2Fill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
-  return <RiAppsFill className='h-3.5 w-3.5 text-components-avatar-shape-fill-stop-100' />
 }

web/app/components/app/create-from-dsl-modal/dsl-confirm-modal.tsx

@@ -0,0 +1,46 @@
+import { useTranslation } from 'react-i18next'
+import Modal from '@/app/components/base/modal'
+import Button from '@/app/components/base/button'
+
+type DSLConfirmModalProps = {
+  versions?: {
+    importedVersion: string
+    systemVersion: string
+  }
+  onCancel: () => void
+  onConfirm: () => void
+  confirmDisabled?: boolean
+}
+const DSLConfirmModal = ({
+  versions = { importedVersion: '', systemVersion: '' },
+  onCancel,
+  onConfirm,
+  confirmDisabled = false,
+}: DSLConfirmModalProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <Modal
+      isShow
+      onClose={() => onCancel()}
+      className='w-[480px]'
+    >
+      <div className='flex flex-col items-start gap-2 self-stretch pb-4'>
+        <div className='title-2xl-semi-bold text-text-primary'>{t('app.newApp.appCreateDSLErrorTitle')}</div>
+        <div className='system-md-regular flex grow flex-col text-text-secondary'>
+          <div>{t('app.newApp.appCreateDSLErrorPart1')}</div>
+          <div>{t('app.newApp.appCreateDSLErrorPart2')}</div>
+          <br />
+          <div>{t('app.newApp.appCreateDSLErrorPart3')}<span className='system-md-medium'>{versions.importedVersion}</span></div>
+          <div>{t('app.newApp.appCreateDSLErrorPart4')}<span className='system-md-medium'>{versions.systemVersion}</span></div>
+        </div>
+      </div>
+      <div className='flex items-start justify-end gap-2 self-stretch pt-6'>
+        <Button variant='secondary' onClick={() => onCancel()}>{t('app.newApp.Cancel')}</Button>
+        <Button variant='primary' destructive onClick={onConfirm} disabled={confirmDisabled}>{t('app.newApp.Confirm')}</Button>
+      </div>
+    </Modal>
+  )
+}
+
+export default DSLConfirmModal

web/app/components/base/icons/src/public/tracing/WeaveIcon.tsx

@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './WeaveIcon.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'WeaveIcon'
+
+export default Icon

web/app/components/base/icons/src/public/tracing/WeaveIconBig.tsx

@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './WeaveIconBig.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'WeaveIconBig'
+
+export default Icon

web/app/components/base/icons/src/public/tracing/index.ts

@@ -5,3 +5,5 @@ export { default as LangsmithIcon } from './LangsmithIcon'
 export { default as OpikIconBig } from './OpikIconBig'
 export { default as OpikIcon } from './OpikIcon'
 export { default as TracingIcon } from './TracingIcon'
+export { default as WeaveIconBig } from './WeaveIconBig'
+export { default as WeaveIcon } from './WeaveIcon'

web/app/components/develop/template/template.zh.mdx

@@ -642,7 +642,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
 
     <CodeGroup title="Response">
     ```json {{ title: 'Response' }}
-    {
     {
       "id": "69d48372-ad81-4c75-9c46-2ce197b4d402",
       "question": "What is your name?",
@@ -650,7 +649,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       "hit_count": 0,
       "created_at": 1735625869
     }
-    }
     ```
     </CodeGroup>
   </Col>
@@ -683,10 +681,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       title="Request"
       tag="PUT"
       label="/apps/annotations/{annotation_id}"
-      targetCode={`curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`}
+      targetCode={`curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`}
     >
     ```bash {{ title: 'cURL' }}
-    curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \
+    curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \
     --header 'Authorization: Bearer {api_key}' \
     --header 'Content-Type: application/json' \
     --data-raw '{
@@ -698,7 +696,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
 
     <CodeGroup title="Response">
     ```json {{ title: 'Response' }}
-    {
     {
       "id": "69d48372-ad81-4c75-9c46-2ce197b4d402",
       "question": "What is your name?",
@@ -706,7 +703,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       "hit_count": 0,
       "created_at": 1735625869
     }
-    }
     ```
     </CodeGroup>
   </Col>
@@ -763,10 +759,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       <Property name='action' type='string' key='action'>
         动作，只能是 'enable' 或 'disable'
       </Property>
-      <Property name='embedding_model_provider' type='string' key='embedding_model_provider'>
+      <Property name='embedding_provider_name' type='string' key='embedding_provider_name'>
         指定的嵌入模型提供商, 必须先在系统内设定好接入的模型，对应的是provider字段
       </Property>
-      <Property name='embedding_model' type='string' key='embedding_model'>
+      <Property name='embedding_model_name' type='string' key='embedding_model_name'>
         指定的嵌入模型，对应的是model字段
       </Property>
       <Property name='score_threshold' type='number' key='score_threshold'>

web/app/components/develop/template/template_advanced_chat.en.mdx

@@ -1336,7 +1336,6 @@ Chat applications support session persistence, allowing previous chat history to
 
     <CodeGroup title="Response">
     ```json {{ title: 'Response' }}
-    {
     {
       "id": "69d48372-ad81-4c75-9c46-2ce197b4d402",
       "question": "What is your name?",
@@ -1344,7 +1343,6 @@ Chat applications support session persistence, allowing previous chat history to
       "hit_count": 0,
       "created_at": 1735625869
     }
-    }
     ```
     </CodeGroup>
   </Col>
@@ -1377,10 +1375,10 @@ Chat applications support session persistence, allowing previous chat history to
       title="Request"
       tag="PUT"
       label="/apps/annotations/{annotation_id}"
-      targetCode={`curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`}
+      targetCode={`curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`}
     >
     ```bash {{ title: 'cURL' }}
-    curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \
+    curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \
     --header 'Authorization: Bearer {api_key}' \
     --header 'Content-Type: application/json' \
     --data-raw '{
@@ -1392,7 +1390,6 @@ Chat applications support session persistence, allowing previous chat history to
 
     <CodeGroup title="Response">
     ```json {{ title: 'Response' }}
-    {
     {
       "id": "69d48372-ad81-4c75-9c46-2ce197b4d402",
       "question": "What is your name?",
@@ -1400,7 +1397,6 @@ Chat applications support session persistence, allowing previous chat history to
       "hit_count": 0,
       "created_at": 1735625869
     }
-    }
     ```
     </CodeGroup>
   </Col>
@@ -1457,10 +1453,10 @@ Chat applications support session persistence, allowing previous chat history to
       <Property name='action' type='string' key='action'>
         Action, can only be 'enable' or 'disable'
       </Property>
-      <Property name='embedding_model_provider' type='string' key='embedding_model_provider'>
+      <Property name='embedding_provider_name' type='string' key='embedding_provider_name'>
         Specified embedding model provider, must be set up in the system first, corresponding to the provider field(Optional)
       </Property>
-      <Property name='embedding_model' type='string' key='embedding_model'>
+      <Property name='embedding_model_name' type='string' key='embedding_model_name'>
         Specified embedding model, corresponding to the model field(Optional)
       </Property>
       <Property name='score_threshold' type='number' key='score_threshold'>

web/app/components/develop/template/template_advanced_chat.zh.mdx

@@ -1360,7 +1360,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
 
     <CodeGroup title="Response">
     ```json {{ title: 'Response' }}
-    {
     {
       "id": "69d48372-ad81-4c75-9c46-2ce197b4d402",
       "question": "What is your name?",
@@ -1368,7 +1367,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       "hit_count": 0,
       "created_at": 1735625869
     }
-    }
     ```
     </CodeGroup>
   </Col>
@@ -1401,10 +1399,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       title="Request"
       tag="PUT"
       label="/apps/annotations/{annotation_id}"
-      targetCode={`curl --location --request POST '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`}
+      targetCode={`curl --location --request PUT '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`}
     >
     ```bash {{ title: 'cURL' }}
-    curl --location --request POST '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \
+    curl --location --request PUT '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \
     --header 'Authorization: Bearer {api_key}' \
     --header 'Content-Type: application/json' \
     --data-raw '{
@@ -1416,7 +1414,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
 
     <CodeGroup title="Response">
     ```json {{ title: 'Response' }}
-    {
     {
       "id": "69d48372-ad81-4c75-9c46-2ce197b4d402",
       "question": "What is your name?",
@@ -1424,7 +1421,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       "hit_count": 0,
       "created_at": 1735625869
     }
-    }
     ```
     </CodeGroup>
   </Col>
@@ -1481,10 +1477,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
       <Property name='action' type='string' key='action'>
         动作，只能是 'enable' 或 'disable'
       </Property>
-      <Property name='embedding_model_provider' type='string' key='embedding_model_provider'>
+      <Property name='embedding_provider_name' type='string' key='embedding_provider_name'>
         指定的嵌入模型提供商, 必须先在系统内设定好接入的模型，对应的是provider字段
       </Property>
-      <Property name='embedding_model' type='string' key='embedding_model'>
+      <Property name='embedding_model_name' type='string' key='embedding_model_name'>
         指定的嵌入模型，对应的是model字段
       </Property>
       <Property name='score_threshold' type='number' key='score_threshold'>

web/app/components/explore/app-list/index.tsx

@@ -1,12 +1,10 @@
 'use client'
 
-import React, { useMemo, useState } from 'react'
+import React, { useCallback, useMemo, useState } from 'react'
-import { useRouter } from 'next/navigation'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import useSWR from 'swr'
 import { useDebounceFn } from 'ahooks'
-import Toast from '../../base/toast'
 import s from './style.module.css'
 import cn from '@/utils/classnames'
 import ExploreContext from '@/context/explore-context'
@@ -14,17 +12,16 @@ import type { App } from '@/models/explore'
 import Category from '@/app/components/explore/category'
 import AppCard from '@/app/components/explore/app-card'
 import { fetchAppDetail, fetchAppList } from '@/service/explore'
-import { importDSL } from '@/service/apps'
 import { useTabSearchParams } from '@/hooks/use-tab-searchparams'
 import CreateAppModal from '@/app/components/explore/create-app-modal'
 import type { CreateAppModalProps } from '@/app/components/explore/create-app-modal'
 import Loading from '@/app/components/base/loading'
-import { NEED_REFRESH_APP_LIST_KEY } from '@/config'
-import { useAppContext } from '@/context/app-context'
-import { getRedirection } from '@/utils/app-redirection'
 import Input from '@/app/components/base/input'
-import { DSLImportMode } from '@/models/app'
+import {
-import { usePluginDependencies } from '@/app/components/workflow/plugin-dependency/hooks'
+  DSLImportMode,
+} from '@/models/app'
+import { useImportDSL } from '@/hooks/use-import-dsl'
+import DSLConfirmModal from '@/app/components/app/create-from-dsl-modal/dsl-confirm-modal'
 
 type AppsProps = {
   onSuccess?: () => void
@@ -39,8 +36,6 @@ const Apps = ({
   onSuccess,
 }: AppsProps) => {
   const { t } = useTranslation()
-  const { isCurrentWorkspaceEditor } = useAppContext()
-  const { push } = useRouter()
   const { hasEditPermission } = useContext(ExploreContext)
   const allCategoriesEn = t('explore.apps.allCategories', { lng: 'en' })
 
@@ -115,7 +110,14 @@ const Apps = ({
 
   const [currApp, setCurrApp] = React.useState<App | null>(null)
   const [isShowCreateModal, setIsShowCreateModal] = React.useState(false)
-  const { handleCheckPluginDependencies } = usePluginDependencies()
+
+  const {
+    handleImportDSL,
+    handleImportDSLConfirm,
+    versions,
+    isFetching,
+  } = useImportDSL()
+  const [showDSLConfirmModal, setShowDSLConfirmModal] = useState(false)
   const onCreate: CreateAppModalProps['onConfirm'] = async ({
     name,
     icon_type,
@@ -123,11 +125,10 @@ const Apps = ({
     icon_background,
     description,
   }) => {
-    const { export_data, mode } = await fetchAppDetail(
+    const { export_data } = await fetchAppDetail(
       currApp?.app.id as string,
     )
-    try {
+    const payload = {
-      const app = await importDSL({
       mode: DSLImportMode.YAML_CONTENT,
       yaml_content: export_data,
       name,
@@ -135,24 +136,23 @@ const Apps = ({
       icon,
       icon_background,
       description,
-      })
+    }
+    await handleImportDSL(payload, {
+      onSuccess: () => {
         setIsShowCreateModal(false)
-      Toast.notify({
+      },
-        type: 'success',
+      onPending: () => {
-        message: t('app.newApp.appCreated'),
+        setShowDSLConfirmModal(true)
+      },
     })
-      if (onSuccess)
-        onSuccess()
-      if (app.app_id)
-        await handleCheckPluginDependencies(app.app_id)
-      localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1')
-      getRedirection(isCurrentWorkspaceEditor, { id: app.app_id!, mode }, push)
-    }
-    catch {
-      Toast.notify({ type: 'error', message: t('app.newApp.appCreateFailed') })
-    }
   }
 
+  const onConfirmDSL = useCallback(async () => {
+    await handleImportDSLConfirm({
+      onSuccess,
+    })
+  }, [handleImportDSLConfirm, onSuccess])
+
   if (!categories || categories.length === 0) {
     return (
       <div className="flex h-full items-center">
@@ -225,9 +225,20 @@ const Apps = ({
           appDescription={currApp?.app.description || ''}
           show={isShowCreateModal}
           onConfirm={onCreate}
+          confirmDisabled={isFetching}
           onHide={() => setIsShowCreateModal(false)}
         />
       )}
+      {
+        showDSLConfirmModal && (
+          <DSLConfirmModal
+            versions={versions}
+            onCancel={() => setShowDSLConfirmModal(false)}
+            onConfirm={onConfirmDSL}
+            confirmDisabled={isFetching}
+          />
+        )
+      }
     </div>
   )
 }

web/app/components/explore/create-app-modal/index.tsx

@@ -35,6 +35,7 @@ export type CreateAppModalProps = {
     description: string
     use_icon_as_answer_icon?: boolean
   }) => Promise<void>
+  confirmDisabled?: boolean
   onHide: () => void
 }
 
@@ -50,6 +51,7 @@ const CreateAppModal = ({
   appMode,
   appUseIconAsAnswerIcon,
   onConfirm,
+  confirmDisabled,
   onHide,
 }: CreateAppModalProps) => {
   const { t } = useTranslation()
@@ -160,7 +162,7 @@ const CreateAppModal = ({
         </div>
         <div className='flex flex-row-reverse'>
           <Button
-            disabled={(!isEditModal && isAppsFull) || !name.trim()}
+            disabled={(!isEditModal && isAppsFull) || !name.trim() || confirmDisabled}
             className='ml-2 w-24 gap-1'
             variant='primary'
             onClick={handleSubmit}

web/app/components/plugins/install-plugin/install-bundle/steps/install.tsx

@@ -8,6 +8,7 @@ import { useTranslation } from 'react-i18next'
 import InstallMulti from './install-multi'
 import { useInstallOrUpdate } from '@/service/use-plugins'
 import useRefreshPluginList from '../../hooks/use-refresh-plugin-list'
+import { useCanInstallPluginFromMarketplace } from '@/app/components/plugins/plugin-page/use-permission'
 const i18nPrefix = 'plugin.installModal'
 
 type Props = {
@@ -74,6 +75,7 @@ const Install: FC<Props> = ({
       installedInfo: installedInfo!,
     })
   }
+  const { canInstallPluginFromMarketplace } = useCanInstallPluginFromMarketplace()
   return (
     <>
       <div className='flex flex-col items-start justify-center gap-4 self-stretch px-6 py-3'>
@@ -101,7 +103,7 @@ const Install: FC<Props> = ({
           <Button
             variant='primary'
             className='flex min-w-[72px] space-x-0.5'
-            disabled={!canInstall || isInstalling || selectedPlugins.length === 0}
+            disabled={!canInstall || isInstalling || selectedPlugins.length === 0 || !canInstallPluginFromMarketplace}
             onClick={handleInstall}
           >
             {isInstalling && <RiLoader2Line className='h-4 w-4 animate-spin-slow' />}

web/app/components/plugins/plugin-page/index.tsx

@@ -144,10 +144,18 @@ const PluginPage = ({
     return activeTab === PLUGIN_PAGE_TABS_MAP.marketplace || values.includes(activeTab)
   }, [activeTab])
 
+  const handleFileChange = (file: File | null) => {
+    if (!file || !file.name.endsWith('.difypkg')) {
+      setCurrentFile(null)
+      return
+    }
+
+    setCurrentFile(file)
+  }
   const uploaderProps = useUploader({
-    onFileChange: setCurrentFile,
+    onFileChange: handleFileChange,
     containerRef,
-    enabled: isPluginsTab,
+    enabled: isPluginsTab && canManagement,
   })
 
   const { dragging, fileUploader, fileChangeHandle, removeFile } = uploaderProps

web/app/components/plugins/plugin-page/use-permission.ts

@@ -3,6 +3,8 @@ import { useAppContext } from '@/context/app-context'
 import Toast from '../../base/toast'
 import { useTranslation } from 'react-i18next'
 import { useInvalidatePermissions, useMutationPermissions, usePermissions } from '@/service/use-plugins'
+import { useSelector as useAppContextSelector } from '@/context/app-context'
+import { useMemo } from 'react'
 
 const hasPermission = (permission: PermissionType | undefined, isAdmin: boolean) => {
   if (!permission)
@@ -43,4 +45,17 @@ const usePermission = () => {
   }
 }
 
+export const useCanInstallPluginFromMarketplace = () => {
+  const { enable_marketplace } = useAppContextSelector(s => s.systemFeatures)
+  const { canManagement } = usePermission()
+
+  const canInstallPluginFromMarketplace = useMemo(() => {
+    return enable_marketplace && canManagement
+  }, [enable_marketplace, canManagement])
+
+  return {
+    canInstallPluginFromMarketplace,
+  }
+}
+
 export default usePermission