diff --git a/README.md b/README.md index 1dc7e2dd98..2909e0e6cf 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ README in বাংলা

-Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features, and more, allowing you to quickly move from prototype to production. +Dify is an open-source platform for developing LLM applications. Its intuitive interface combines agentic AI workflows, RAG pipelines, agent capabilities, model management, observability features, and more—allowing you to quickly move from prototype to production. ## Quick start @@ -65,7 +65,7 @@ Dify is an open-source LLM app development platform. Its intuitive interface com
-The easiest way to start the Dify server is through [docker compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine: +The easiest way to start the Dify server is through [Docker Compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine: ```bash cd dify @@ -205,6 +205,7 @@ If you'd like to configure a highly-available setup, there are community-contrib - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Using Terraform for Deployment @@ -261,8 +262,8 @@ At the same time, please consider supporting Dify by sharing it on social media ## Security disclosure -To protect your privacy, please avoid posting security issues on GitHub. Instead, send your questions to security@dify.ai and we will provide you with a more detailed answer. +To protect your privacy, please avoid posting security issues on GitHub. Instead, report issues to security@dify.ai, and our team will respond with detailed answer. ## License -This repository is available under the [Dify Open Source License](LICENSE), which is essentially Apache 2.0 with a few additional restrictions. +This repository is licensed under the [Dify Open Source License](LICENSE), based on Apache 2.0 with additional conditions. diff --git a/README_AR.md b/README_AR.md index d93bca8646..e959ca0f78 100644 --- a/README_AR.md +++ b/README_AR.md @@ -188,6 +188,7 @@ docker compose up -d - [رسم بياني Helm من قبل @magicsong](https://github.com/magicsong/ai-charts) - [ملف YAML من قبل @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [ملف YAML من قبل @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 جديد! ملفات YAML (تدعم Dify v1.6.0) بواسطة @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### استخدام Terraform للتوزيع diff --git a/README_BN.md b/README_BN.md index 3efee3684d..29d7374ea5 100644 --- a/README_BN.md +++ b/README_BN.md @@ -204,6 +204,8 @@ GitHub-এ ডিফাইকে স্টার দিয়ে রাখুন - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 নতুন! YAML ফাইলসমূহ (Dify v1.6.0 সমর্থিত) তৈরি করেছেন @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) + #### টেরাফর্ম ব্যবহার করে ডিপ্লয় diff --git a/README_CN.md b/README_CN.md index 21e27429ec..486a368c09 100644 --- a/README_CN.md +++ b/README_CN.md @@ -194,9 +194,9 @@ docker compose up -d 如果您需要自定义配置，请参考 [.env.example](docker/.env.example) 文件中的注释，并更新 `.env` 文件中对应的值。此外，您可能需要根据您的具体部署环境和需求对 `docker-compose.yaml` 文件本身进行调整，例如更改镜像版本、端口映射或卷挂载。完成任何更改后，请重新运行 `docker-compose up -d`。您可以在[此处](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用环境变量的完整列表。 -#### 使用 Helm Chart 部署 +#### 使用 Helm Chart 或 Kubernetes 资源清单（YAML）部署 -使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。 +使用 [Helm Chart](https://helm.sh/) 版本或者 Kubernetes 资源清单（YAML），可以在 Kubernetes 上部署 Dify。 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) @@ -204,6 +204,10 @@ docker compose up -d - [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML 文件 (支持 Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) + + + #### 使用 Terraform 部署 使用 [terraform](https://www.terraform.io/) 一键将 Dify 部署到云平台 diff --git a/README_DE.md b/README_DE.md index 20c313035e..fce52c34c2 100644 --- a/README_DE.md +++ b/README_DE.md @@ -203,6 +203,7 @@ Falls Sie eine hochverfügbare Konfiguration einrichten möchten, gibt es von de - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Terraform für die Bereitstellung verwenden diff --git a/README_ES.md b/README_ES.md index e4b7df6686..6fd6dfcee8 100644 --- a/README_ES.md +++ b/README_ES.md @@ -203,6 +203,7 @@ Si desea configurar una configuración de alta disponibilidad, la comunidad prop - [Gráfico Helm por @magicsong](https://github.com/magicsong/ai-charts) - [Ficheros YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [Ficheros YAML por @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 ¡NUEVO! Archivos YAML (compatible con Dify v1.6.0) por @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Uso de Terraform para el despliegue diff --git a/README_FR.md b/README_FR.md index 8fd17fb7c3..b2209fb495 100644 --- a/README_FR.md +++ b/README_FR.md @@ -201,6 +201,7 @@ Si vous souhaitez configurer une configuration haute disponibilité, la communau - [Helm Chart par @magicsong](https://github.com/magicsong/ai-charts) - [Fichier YAML par @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [Fichier YAML par @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NOUVEAU ! Fichiers YAML (compatible avec Dify v1.6.0) par @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Utilisation de Terraform pour le déploiement diff --git a/README_JA.md b/README_JA.md index a3ee81e1f2..c658225f90 100644 --- a/README_JA.md +++ b/README_JA.md @@ -202,6 +202,7 @@ docker compose up -d - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 新着！YAML ファイル（Dify v1.6.0 対応）by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Terraformを使用したデプロイ diff --git a/README_KL.md b/README_KL.md index 3e5ab1a74f..bfafcc7407 100644 --- a/README_KL.md +++ b/README_KL.md @@ -201,6 +201,7 @@ If you'd like to configure a highly-available setup, there are community-contrib - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Terraform atorlugu pilersitsineq diff --git a/README_KR.md b/README_KR.md index 3c504900e1..282117e776 100644 --- a/README_KR.md +++ b/README_KR.md @@ -195,6 +195,7 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Terraform을 사용한 배포 diff --git a/README_PT.md b/README_PT.md index fb5f3662ae..576f6b48f7 100644 --- a/README_PT.md +++ b/README_PT.md @@ -200,6 +200,7 @@ Se deseja configurar uma instalação de alta disponibilidade, há [Helm Charts] - [Helm Chart de @magicsong](https://github.com/magicsong/ai-charts) - [Arquivo YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [Arquivo YAML por @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NOVO! Arquivos YAML (Compatível com Dify v1.6.0) por @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Usando o Terraform para Implantação diff --git a/README_SI.md b/README_SI.md index 647069a220..7ded001d86 100644 --- a/README_SI.md +++ b/README_SI.md @@ -201,6 +201,7 @@ Star Dify on GitHub and be instantly notified of new releases. - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Uporaba Terraform za uvajanje diff --git a/README_TR.md b/README_TR.md index f52335646a..6e94e54fa0 100644 --- a/README_TR.md +++ b/README_TR.md @@ -194,6 +194,7 @@ Yüksek kullanılabilirliğe sahip bir kurulum yapılandırmak isterseniz, Dify' - [@BorisPolonsky tarafından Helm Chart](https://github.com/BorisPolonsky/dify-helm) - [@Winson-030 tarafından YAML dosyası](https://github.com/Winson-030/dify-kubernetes) - [@wyy-holding tarafından YAML dosyası](https://github.com/wyy-holding/dify-k8s) +- [🚀 YENİ! YAML dosyaları (Dify v1.6.0 destekli) @Zhoneym tarafından](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Dağıtım için Terraform Kullanımı diff --git a/README_TW.md b/README_TW.md index 71082ff893..6e3e22b5c1 100644 --- a/README_TW.md +++ b/README_TW.md @@ -197,12 +197,13 @@ Dify 的所有功能都提供相應的 API，因此您可以輕鬆地將 Dify 如果您需要自定義配置，請參考我們的 [.env.example](docker/.env.example) 文件中的註釋，並在您的 `.env` 文件中更新相應的值。此外，根據您特定的部署環境和需求，您可能需要調整 `docker-compose.yaml` 文件本身，例如更改映像版本、端口映射或卷掛載。進行任何更改後，請重新運行 `docker-compose up -d`。您可以在[這裡](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用環境變數的完整列表。 -如果您想配置高可用性設置，社區貢獻的 [Helm Charts](https://helm.sh/) 和 YAML 文件允許在 Kubernetes 上部署 Dify。 +如果您想配置高可用性設置，社區貢獻的 [Helm Charts](https://helm.sh/) 和 Kubernetes 資源清單（YAML）允許在 Kubernetes 上部署 Dify。 - [由 @LeoQuote 提供的 Helm Chart](https://github.com/douban/charts/tree/master/charts/dify) - [由 @BorisPolonsky 提供的 Helm Chart](https://github.com/BorisPolonsky/dify-helm) - [由 @Winson-030 提供的 YAML 文件](https://github.com/Winson-030/dify-kubernetes) - [由 @wyy-holding 提供的 YAML 文件](https://github.com/wyy-holding/dify-k8s) +- [🚀 NEW! YAML 檔案（支援 Dify v1.6.0）by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) ### 使用 Terraform 進行部署 diff --git a/README_VI.md b/README_VI.md index 58d8434fff..51314e6de5 100644 --- a/README_VI.md +++ b/README_VI.md @@ -196,6 +196,7 @@ Nếu bạn muốn cấu hình một cài đặt có độ sẵn sàng cao, có - [Helm Chart bởi @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) - [Tệp YAML bởi @Winson-030](https://github.com/Winson-030/dify-kubernetes) - [Tệp YAML bởi @wyy-holding](https://github.com/wyy-holding/dify-k8s) +- [🚀 MỚI! Tệp YAML (Hỗ trợ Dify v1.6.0) bởi @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes) #### Sử dụng Terraform để Triển khai diff --git a/api/.env.example b/api/.env.example index baa9c382c8..eab017a624 100644 --- a/api/.env.example +++ b/api/.env.example @@ -17,6 +17,11 @@ APP_WEB_URL=http://127.0.0.1:3000 # Files URL FILES_URL=http://127.0.0.1:5001 +# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network. +# Set this to the internal Docker service URL for proper plugin file access. +# Example: INTERNAL_FILES_URL=http://api:5001 +INTERNAL_FILES_URL=http://127.0.0.1:5001 + # The time in seconds after the signature is rejected FILES_ACCESS_TIMEOUT=300 @@ -444,6 +449,19 @@ MAX_VARIABLE_SIZE=204800 # hybrid: Save new data to object storage, read from both object storage and RDBMS WORKFLOW_NODE_EXECUTION_STORAGE=rdbms +# Repository configuration +# Core workflow execution repository implementation +CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository + +# Core workflow node execution repository implementation +CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository + +# API workflow node execution repository implementation +API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository + +# API workflow run repository implementation +API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository + # App configuration APP_MAX_EXECUTION_TIME=1200 APP_MAX_ACTIVE_REQUESTS=0 diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index df15b92c35..f6a8b037ca 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -237,6 +237,13 @@ class FileAccessConfig(BaseSettings): default="", ) + INTERNAL_FILES_URL: str = Field( + description="Internal base URL for file access within Docker network," + " used for plugin daemon and internal service communication." + " Falls back to FILES_URL if not specified.", + default="", + ) + FILES_ACCESS_TIMEOUT: int = Field( description="Expiration time in seconds for file access URLs", default=300, @@ -530,6 +537,33 @@ class WorkflowNodeExecutionConfig(BaseSettings): ) +class RepositoryConfig(BaseSettings): + """ + Configuration for repository implementations + """ + + CORE_WORKFLOW_EXECUTION_REPOSITORY: str = Field( + description="Repository implementation for WorkflowExecution. Specify as a module path", + default="core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository", + ) + + CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: str = Field( + description="Repository implementation for WorkflowNodeExecution. Specify as a module path", + default="core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository", + ) + + API_WORKFLOW_NODE_EXECUTION_REPOSITORY: str = Field( + description="Service-layer repository implementation for WorkflowNodeExecutionModel operations. " + "Specify as a module path", + default="repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository", + ) + + API_WORKFLOW_RUN_REPOSITORY: str = Field( + description="Service-layer repository implementation for WorkflowRun operations. Specify as a module path", + default="repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository", + ) + + class AuthConfig(BaseSettings): """ Configuration for authentication and OAuth @@ -896,6 +930,7 @@ class FeatureConfig( MultiModalTransferConfig, PositionConfig, RagEtlConfig, + RepositoryConfig, SecurityConfig, ToolConfig, UpdateConfig, diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index dbdcdc46ce..e25f92399c 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -56,6 +56,7 @@ from .app import ( conversation, conversation_variables, generator, + mcp_server, message, model_config, ops_trace, diff --git a/api/controllers/console/app/mcp_server.py b/api/controllers/console/app/mcp_server.py new file mode 100644 index 0000000000..0f53860f56 --- /dev/null +++ b/api/controllers/console/app/mcp_server.py @@ -0,0 +1,107 @@ +import json +from enum import StrEnum + +from flask_login import current_user +from flask_restful import Resource, marshal_with, reqparse +from werkzeug.exceptions import NotFound + +from controllers.console import api +from controllers.console.app.wraps import get_app_model +from controllers.console.wraps import account_initialization_required, setup_required +from extensions.ext_database import db +from fields.app_fields import app_server_fields +from libs.login import login_required +from models.model import AppMCPServer + + +class AppMCPServerStatus(StrEnum): + ACTIVE = "active" + INACTIVE = "inactive" + + +class AppMCPServerController(Resource): + @setup_required + @login_required + @account_initialization_required + @get_app_model + @marshal_with(app_server_fields) + def get(self, app_model): + server = db.session.query(AppMCPServer).filter(AppMCPServer.app_id == app_model.id).first() + return server + + @setup_required + @login_required + @account_initialization_required + @get_app_model + @marshal_with(app_server_fields) + def post(self, app_model): + # The role of the current user in the ta table must be editor, admin, or owner + if not current_user.is_editor: + raise NotFound() + parser = reqparse.RequestParser() + parser.add_argument("description", type=str, required=True, location="json") + parser.add_argument("parameters", type=dict, required=True, location="json") + args = parser.parse_args() + server = AppMCPServer( + name=app_model.name, + description=args["description"], + parameters=json.dumps(args["parameters"], ensure_ascii=False), + status=AppMCPServerStatus.ACTIVE, + app_id=app_model.id, + tenant_id=current_user.current_tenant_id, + server_code=AppMCPServer.generate_server_code(16), + ) + db.session.add(server) + db.session.commit() + return server + + @setup_required + @login_required + @account_initialization_required + @get_app_model + @marshal_with(app_server_fields) + def put(self, app_model): + if not current_user.is_editor: + raise NotFound() + parser = reqparse.RequestParser() + parser.add_argument("id", type=str, required=True, location="json") + parser.add_argument("description", type=str, required=True, location="json") + parser.add_argument("parameters", type=dict, required=True, location="json") + parser.add_argument("status", type=str, required=False, location="json") + args = parser.parse_args() + server = db.session.query(AppMCPServer).filter(AppMCPServer.id == args["id"]).first() + if not server: + raise NotFound() + server.description = args["description"] + server.parameters = json.dumps(args["parameters"], ensure_ascii=False) + if args["status"]: + if args["status"] not in [status.value for status in AppMCPServerStatus]: + raise ValueError("Invalid status") + server.status = args["status"] + db.session.commit() + return server + + +class AppMCPServerRefreshController(Resource): + @setup_required + @login_required + @account_initialization_required + @marshal_with(app_server_fields) + def get(self, server_id): + if not current_user.is_editor: + raise NotFound() + server = ( + db.session.query(AppMCPServer) + .filter(AppMCPServer.id == server_id) + .filter(AppMCPServer.tenant_id == current_user.current_tenant_id) + .first() + ) + if not server: + raise NotFound() + server.server_code = AppMCPServer.generate_server_code(16) + db.session.commit() + return server + + +api.add_resource(AppMCPServerController, "/apps//server") +api.add_resource(AppMCPServerRefreshController, "/apps//server/refresh") diff --git a/api/controllers/console/app/statistic.py b/api/controllers/console/app/statistic.py index 86aed77412..32b64d10c5 100644 --- a/api/controllers/console/app/statistic.py +++ b/api/controllers/console/app/statistic.py @@ -2,6 +2,7 @@ from datetime import datetime from decimal import Decimal import pytz +import sqlalchemy as sa from flask import jsonify from flask_login import current_user from flask_restful import Resource, reqparse @@ -9,10 +10,11 @@ from flask_restful import Resource, reqparse from controllers.console import api from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, setup_required +from core.app.entities.app_invoke_entities import InvokeFrom from extensions.ext_database import db from libs.helper import DatetimeString from libs.login import login_required -from models.model import AppMode +from models import AppMode, Message class DailyMessageStatistic(Resource): @@ -85,46 +87,41 @@ class DailyConversationStatistic(Resource): parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args") args = parser.parse_args() - sql_query = """SELECT - DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, - COUNT(DISTINCT messages.conversation_id) AS conversation_count -FROM - messages -WHERE - app_id = :app_id""" - arg_dict = {"tz": account.timezone, "app_id": app_model.id} - timezone = pytz.timezone(account.timezone) utc_timezone = pytz.utc + stmt = ( + sa.select( + sa.func.date( + sa.func.date_trunc("day", sa.text("created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz")) + ).label("date"), + sa.func.count(sa.distinct(Message.conversation_id)).label("conversation_count"), + ) + .select_from(Message) + .where(Message.app_id == app_model.id, Message.invoke_from != InvokeFrom.DEBUGGER.value) + ) + if args["start"]: start_datetime = datetime.strptime(args["start"], "%Y-%m-%d %H:%M") start_datetime = start_datetime.replace(second=0) - start_datetime_timezone = timezone.localize(start_datetime) start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone) - - sql_query += " AND created_at >= :start" - arg_dict["start"] = start_datetime_utc + stmt = stmt.where(Message.created_at >= start_datetime_utc) if args["end"]: end_datetime = datetime.strptime(args["end"], "%Y-%m-%d %H:%M") end_datetime = end_datetime.replace(second=0) - end_datetime_timezone = timezone.localize(end_datetime) end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone) + stmt = stmt.where(Message.created_at < end_datetime_utc) - sql_query += " AND created_at < :end" - arg_dict["end"] = end_datetime_utc - - sql_query += " GROUP BY date ORDER BY date" + stmt = stmt.group_by("date").order_by("date") response_data = [] - with db.engine.begin() as conn: - rs = conn.execute(db.text(sql_query), arg_dict) - for i in rs: - response_data.append({"date": str(i.date), "conversation_count": i.conversation_count}) + rs = conn.execute(stmt, {"tz": account.timezone}) + for row in rs: + response_data.append({"date": str(row.date), "conversation_count": row.conversation_count}) return jsonify({"data": response_data}) diff --git a/api/controllers/console/app/wraps.py b/api/controllers/console/app/wraps.py index 03b60610aa..3322350e25 100644 --- a/api/controllers/console/app/wraps.py +++ b/api/controllers/console/app/wraps.py @@ -35,8 +35,6 @@ def get_app_model(view: Optional[Callable] = None, *, mode: Union[AppMode, list[ raise AppNotFoundError() app_mode = AppMode.value_of(app_model.mode) - if app_mode == AppMode.CHANNEL: - raise AppNotFoundError() if mode is not None: if isinstance(mode, list): diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index 2b1379bfb2..df50871a38 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -1,6 +1,7 @@ import io +from urllib.parse import urlparse -from flask import send_file +from flask import redirect, send_file from flask_login import current_user from flask_restful import Resource, reqparse from sqlalchemy.orm import Session @@ -9,17 +10,34 @@ from werkzeug.exceptions import Forbidden from configs import dify_config from controllers.console import api from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required +from core.mcp.auth.auth_flow import auth, handle_callback +from core.mcp.auth.auth_provider import OAuthClientProvider +from core.mcp.error import MCPAuthError, MCPError +from core.mcp.mcp_client import MCPClient from core.model_runtime.utils.encoders import jsonable_encoder from extensions.ext_database import db from libs.helper import alphanumeric, uuid_value from libs.login import login_required from services.tools.api_tools_manage_service import ApiToolManageService from services.tools.builtin_tools_manage_service import BuiltinToolManageService +from services.tools.mcp_tools_mange_service import MCPToolManageService from services.tools.tool_labels_service import ToolLabelsService from services.tools.tools_manage_service import ToolCommonService +from services.tools.tools_transform_service import ToolTransformService from services.tools.workflow_tools_manage_service import WorkflowToolManageService +def is_valid_url(url: str) -> bool: + if not url: + return False + + try: + parsed = urlparse(url) + return all([parsed.scheme, parsed.netloc]) and parsed.scheme in ["http", "https"] + except Exception: + return False + + class ToolProviderListApi(Resource): @setup_required @login_required @@ -34,7 +52,7 @@ class ToolProviderListApi(Resource): req.add_argument( "type", type=str, - choices=["builtin", "model", "api", "workflow"], + choices=["builtin", "model", "api", "workflow", "mcp"], required=False, nullable=True, location="args", @@ -613,6 +631,166 @@ class ToolLabelsApi(Resource): return jsonable_encoder(ToolLabelsService.list_tool_labels()) +class ToolProviderMCPApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("server_url", type=str, required=True, nullable=False, location="json") + parser.add_argument("name", type=str, required=True, nullable=False, location="json") + parser.add_argument("icon", type=str, required=True, nullable=False, location="json") + parser.add_argument("icon_type", type=str, required=True, nullable=False, location="json") + parser.add_argument("icon_background", type=str, required=False, nullable=True, location="json", default="") + parser.add_argument("server_identifier", type=str, required=True, nullable=False, location="json") + args = parser.parse_args() + user = current_user + if not is_valid_url(args["server_url"]): + raise ValueError("Server URL is not valid.") + return jsonable_encoder( + MCPToolManageService.create_mcp_provider( + tenant_id=user.current_tenant_id, + server_url=args["server_url"], + name=args["name"], + icon=args["icon"], + icon_type=args["icon_type"], + icon_background=args["icon_background"], + user_id=user.id, + server_identifier=args["server_identifier"], + ) + ) + + @setup_required + @login_required + @account_initialization_required + def put(self): + parser = reqparse.RequestParser() + parser.add_argument("server_url", type=str, required=True, nullable=False, location="json") + parser.add_argument("name", type=str, required=True, nullable=False, location="json") + parser.add_argument("icon", type=str, required=True, nullable=False, location="json") + parser.add_argument("icon_type", type=str, required=True, nullable=False, location="json") + parser.add_argument("icon_background", type=str, required=False, nullable=True, location="json") + parser.add_argument("provider_id", type=str, required=True, nullable=False, location="json") + parser.add_argument("server_identifier", type=str, required=True, nullable=False, location="json") + args = parser.parse_args() + if not is_valid_url(args["server_url"]): + if "[__HIDDEN__]" in args["server_url"]: + pass + else: + raise ValueError("Server URL is not valid.") + MCPToolManageService.update_mcp_provider( + tenant_id=current_user.current_tenant_id, + provider_id=args["provider_id"], + server_url=args["server_url"], + name=args["name"], + icon=args["icon"], + icon_type=args["icon_type"], + icon_background=args["icon_background"], + server_identifier=args["server_identifier"], + ) + return {"result": "success"} + + @setup_required + @login_required + @account_initialization_required + def delete(self): + parser = reqparse.RequestParser() + parser.add_argument("provider_id", type=str, required=True, nullable=False, location="json") + args = parser.parse_args() + MCPToolManageService.delete_mcp_tool(tenant_id=current_user.current_tenant_id, provider_id=args["provider_id"]) + return {"result": "success"} + + +class ToolMCPAuthApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("provider_id", type=str, required=True, nullable=False, location="json") + parser.add_argument("authorization_code", type=str, required=False, nullable=True, location="json") + args = parser.parse_args() + provider_id = args["provider_id"] + tenant_id = current_user.current_tenant_id + provider = MCPToolManageService.get_mcp_provider_by_provider_id(provider_id, tenant_id) + if not provider: + raise ValueError("provider not found") + try: + with MCPClient( + provider.decrypted_server_url, + provider_id, + tenant_id, + authed=False, + authorization_code=args["authorization_code"], + for_list=True, + ): + MCPToolManageService.update_mcp_provider_credentials( + mcp_provider=provider, + credentials=provider.decrypted_credentials, + authed=True, + ) + return {"result": "success"} + + except MCPAuthError: + auth_provider = OAuthClientProvider(provider_id, tenant_id, for_list=True) + return auth(auth_provider, provider.decrypted_server_url, args["authorization_code"]) + except MCPError as e: + MCPToolManageService.update_mcp_provider_credentials( + mcp_provider=provider, + credentials={}, + authed=False, + ) + raise ValueError(f"Failed to connect to MCP server: {e}") from e + + +class ToolMCPDetailApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, provider_id): + user = current_user + provider = MCPToolManageService.get_mcp_provider_by_provider_id(provider_id, user.current_tenant_id) + return jsonable_encoder(ToolTransformService.mcp_provider_to_user_provider(provider, for_list=True)) + + +class ToolMCPListAllApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self): + user = current_user + tenant_id = user.current_tenant_id + + tools = MCPToolManageService.retrieve_mcp_tools(tenant_id=tenant_id) + + return [tool.to_dict() for tool in tools] + + +class ToolMCPUpdateApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, provider_id): + tenant_id = current_user.current_tenant_id + tools = MCPToolManageService.list_mcp_tool_from_remote_server( + tenant_id=tenant_id, + provider_id=provider_id, + ) + return jsonable_encoder(tools) + + +class ToolMCPCallbackApi(Resource): + def get(self): + parser = reqparse.RequestParser() + parser.add_argument("code", type=str, required=True, nullable=False, location="args") + parser.add_argument("state", type=str, required=True, nullable=False, location="args") + args = parser.parse_args() + state_key = args["state"] + authorization_code = args["code"] + handle_callback(state_key, authorization_code) + return redirect(f"{dify_config.CONSOLE_WEB_URL}/oauth-callback") + + # tool provider api.add_resource(ToolProviderListApi, "/workspaces/current/tool-providers") @@ -647,8 +825,15 @@ api.add_resource(ToolWorkflowProviderDeleteApi, "/workspaces/current/tool-provid api.add_resource(ToolWorkflowProviderGetApi, "/workspaces/current/tool-provider/workflow/get") api.add_resource(ToolWorkflowProviderListToolApi, "/workspaces/current/tool-provider/workflow/tools") +# mcp tool provider +api.add_resource(ToolMCPDetailApi, "/workspaces/current/tool-provider/mcp/tools/") +api.add_resource(ToolProviderMCPApi, "/workspaces/current/tool-provider/mcp") +api.add_resource(ToolMCPUpdateApi, "/workspaces/current/tool-provider/mcp/update/") +api.add_resource(ToolMCPAuthApi, "/workspaces/current/tool-provider/mcp/auth") +api.add_resource(ToolMCPCallbackApi, "/mcp/oauth/callback") + api.add_resource(ToolBuiltinListApi, "/workspaces/current/tools/builtin") api.add_resource(ToolApiListApi, "/workspaces/current/tools/api") +api.add_resource(ToolMCPListAllApi, "/workspaces/current/tools/mcp") api.add_resource(ToolWorkflowListApi, "/workspaces/current/tools/workflow") - api.add_resource(ToolLabelsApi, "/workspaces/current/tool-labels") diff --git a/api/controllers/mcp/__init__.py b/api/controllers/mcp/__init__.py new file mode 100644 index 0000000000..1b3e0a5621 --- /dev/null +++ b/api/controllers/mcp/__init__.py @@ -0,0 +1,8 @@ +from flask import Blueprint + +from libs.external_api import ExternalApi + +bp = Blueprint("mcp", __name__, url_prefix="/mcp") +api = ExternalApi(bp) + +from . import mcp diff --git a/api/controllers/mcp/mcp.py b/api/controllers/mcp/mcp.py new file mode 100644 index 0000000000..ead728bfb0 --- /dev/null +++ b/api/controllers/mcp/mcp.py @@ -0,0 +1,104 @@ +from flask_restful import Resource, reqparse +from pydantic import ValidationError + +from controllers.console.app.mcp_server import AppMCPServerStatus +from controllers.mcp import api +from core.app.app_config.entities import VariableEntity +from core.mcp import types +from core.mcp.server.streamable_http import MCPServerStreamableHTTPRequestHandler +from core.mcp.types import ClientNotification, ClientRequest +from core.mcp.utils import create_mcp_error_response +from extensions.ext_database import db +from libs import helper +from models.model import App, AppMCPServer, AppMode + + +class MCPAppApi(Resource): + def post(self, server_code): + def int_or_str(value): + if isinstance(value, (int, str)): + return value + else: + return None + + parser = reqparse.RequestParser() + parser.add_argument("jsonrpc", type=str, required=True, location="json") + parser.add_argument("method", type=str, required=True, location="json") + parser.add_argument("params", type=dict, required=False, location="json") + parser.add_argument("id", type=int_or_str, required=False, location="json") + args = parser.parse_args() + + request_id = args.get("id") + + server = db.session.query(AppMCPServer).filter(AppMCPServer.server_code == server_code).first() + if not server: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_REQUEST, "Server Not Found") + ) + + if server.status != AppMCPServerStatus.ACTIVE: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_REQUEST, "Server is not active") + ) + + app = db.session.query(App).filter(App.id == server.app_id).first() + if not app: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_REQUEST, "App Not Found") + ) + + if app.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}: + workflow = app.workflow + if workflow is None: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_REQUEST, "App is unavailable") + ) + + user_input_form = workflow.user_input_form(to_old_structure=True) + else: + app_model_config = app.app_model_config + if app_model_config is None: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_REQUEST, "App is unavailable") + ) + + features_dict = app_model_config.to_dict() + user_input_form = features_dict.get("user_input_form", []) + converted_user_input_form: list[VariableEntity] = [] + try: + for item in user_input_form: + variable_type = item.get("type", "") or list(item.keys())[0] + variable = item[variable_type] + converted_user_input_form.append( + VariableEntity( + type=variable_type, + variable=variable.get("variable"), + description=variable.get("description") or "", + label=variable.get("label"), + required=variable.get("required", False), + max_length=variable.get("max_length"), + options=variable.get("options") or [], + ) + ) + except ValidationError as e: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_PARAMS, f"Invalid user_input_form: {str(e)}") + ) + + try: + request: ClientRequest | ClientNotification = ClientRequest.model_validate(args) + except ValidationError as e: + try: + notification = ClientNotification.model_validate(args) + request = notification + except ValidationError as e: + return helper.compact_generate_response( + create_mcp_error_response(request_id, types.INVALID_PARAMS, f"Invalid MCP request: {str(e)}") + ) + + mcp_server_handler = MCPServerStreamableHTTPRequestHandler(app, request, converted_user_input_form) + response = mcp_server_handler.handle() + return helper.compact_generate_response(response) + + +api.add_resource(MCPAppApi, "/server//mcp") diff --git a/api/controllers/service_api/app/workflow.py b/api/controllers/service_api/app/workflow.py index efb4acc5fb..ac2ebf2b09 100644 --- a/api/controllers/service_api/app/workflow.py +++ b/api/controllers/service_api/app/workflow.py @@ -3,7 +3,7 @@ import logging from dateutil.parser import isoparse from flask_restful import Resource, fields, marshal_with, reqparse from flask_restful.inputs import int_range -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from werkzeug.exceptions import InternalServerError from controllers.service_api import api @@ -30,7 +30,7 @@ from fields.workflow_app_log_fields import workflow_app_log_pagination_fields from libs import helper from libs.helper import TimestampField from models.model import App, AppMode, EndUser -from models.workflow import WorkflowRun +from repositories.factory import DifyAPIRepositoryFactory from services.app_generate_service import AppGenerateService from services.errors.llm import InvokeRateLimitError from services.workflow_app_service import WorkflowAppService @@ -63,7 +63,15 @@ class WorkflowRunDetailApi(Resource): if app_mode not in [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]: raise NotWorkflowAppError() - workflow_run = db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_run_id).first() + # Use repository to get workflow run + session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + workflow_run_repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker) + + workflow_run = workflow_run_repo.get_workflow_run_by_id( + tenant_id=app_model.tenant_id, + app_id=app_model.id, + run_id=workflow_run_id, + ) return workflow_run diff --git a/api/core/agent/base_agent_runner.py b/api/core/agent/base_agent_runner.py index 6998e4d29a..28bf4a9a23 100644 --- a/api/core/agent/base_agent_runner.py +++ b/api/core/agent/base_agent_runner.py @@ -3,6 +3,8 @@ import logging import uuid from typing import Optional, Union, cast +from sqlalchemy import select + from core.agent.entities import AgentEntity, AgentToolEntity from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfig @@ -161,10 +163,14 @@ class BaseAgentRunner(AppRunner): if parameter.type == ToolParameter.ToolParameterType.SELECT: enum = [option.value for option in parameter.options] if parameter.options else [] - message_tool.parameters["properties"][parameter.name] = { - "type": parameter_type, - "description": parameter.llm_description or "", - } + message_tool.parameters["properties"][parameter.name] = ( + { + "type": parameter_type, + "description": parameter.llm_description or "", + } + if parameter.input_schema is None + else parameter.input_schema + ) if len(enum) > 0: message_tool.parameters["properties"][parameter.name]["enum"] = enum @@ -254,10 +260,14 @@ class BaseAgentRunner(AppRunner): if parameter.type == ToolParameter.ToolParameterType.SELECT: enum = [option.value for option in parameter.options] if parameter.options else [] - prompt_tool.parameters["properties"][parameter.name] = { - "type": parameter_type, - "description": parameter.llm_description or "", - } + prompt_tool.parameters["properties"][parameter.name] = ( + { + "type": parameter_type, + "description": parameter.llm_description or "", + } + if parameter.input_schema is None + else parameter.input_schema + ) if len(enum) > 0: prompt_tool.parameters["properties"][parameter.name]["enum"] = enum @@ -409,12 +419,15 @@ class BaseAgentRunner(AppRunner): if isinstance(prompt_message, SystemPromptMessage): result.append(prompt_message) - messages: list[Message] = ( - db.session.query(Message) - .filter( - Message.conversation_id == self.message.conversation_id, + messages = ( + ( + db.session.execute( + select(Message) + .where(Message.conversation_id == self.message.conversation_id) + .order_by(Message.created_at.desc()) + ) ) - .order_by(Message.created_at.desc()) + .scalars() .all() ) diff --git a/api/core/agent/plugin_entities.py b/api/core/agent/plugin_entities.py index 9c722baa23..3b48288710 100644 --- a/api/core/agent/plugin_entities.py +++ b/api/core/agent/plugin_entities.py @@ -85,7 +85,7 @@ class AgentStrategyEntity(BaseModel): description: I18nObject = Field(..., description="The description of the agent strategy") output_schema: Optional[dict] = None features: Optional[list[AgentFeature]] = None - + meta_version: Optional[str] = None # pydantic configs model_config = ConfigDict(protected_namespaces=()) diff --git a/api/core/agent/strategy/plugin.py b/api/core/agent/strategy/plugin.py index 79b074cf95..4cfcfbf86a 100644 --- a/api/core/agent/strategy/plugin.py +++ b/api/core/agent/strategy/plugin.py @@ -15,10 +15,12 @@ class PluginAgentStrategy(BaseAgentStrategy): tenant_id: str declaration: AgentStrategyEntity + meta_version: str | None = None - def __init__(self, tenant_id: str, declaration: AgentStrategyEntity): + def __init__(self, tenant_id: str, declaration: AgentStrategyEntity, meta_version: str | None): self.tenant_id = tenant_id self.declaration = declaration + self.meta_version = meta_version def get_parameters(self) -> Sequence[AgentStrategyParameter]: return self.declaration.parameters diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 7877408cef..4b8f5ebe27 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -25,8 +25,7 @@ from core.app.entities.task_entities import ChatbotAppBlockingResponse, ChatbotA from core.model_runtime.errors.invoke import InvokeAuthorizationError from core.ops.ops_trace_manager import TraceQueueManager from core.prompt.utils.get_thread_messages_length import get_thread_messages_length -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository -from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.workflow.repositories.draft_variable_repository import ( DraftVariableSaverFactory, ) @@ -183,14 +182,14 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): workflow_triggered_from = WorkflowRunTriggeredFrom.DEBUGGING else: workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN - workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository( + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=workflow_triggered_from, ) # Create workflow node execution repository - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, @@ -260,14 +259,14 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository - workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository( + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, ) # Create workflow node execution repository - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, @@ -343,14 +342,14 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository - workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository( + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, ) # Create workflow node execution repository - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index 40a1e272a7..2f9632e97d 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -23,8 +23,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerat from core.app.entities.task_entities import WorkflowAppBlockingResponse, WorkflowAppStreamResponse from core.model_runtime.errors.invoke import InvokeAuthorizationError from core.ops.ops_trace_manager import TraceQueueManager -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository -from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository @@ -156,14 +155,14 @@ class WorkflowAppGenerator(BaseAppGenerator): workflow_triggered_from = WorkflowRunTriggeredFrom.DEBUGGING else: workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN - workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository( + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=workflow_triggered_from, ) # Create workflow node execution repository - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, @@ -306,16 +305,14 @@ class WorkflowAppGenerator(BaseAppGenerator): # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository - workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository( + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, ) # Create workflow node execution repository - session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) - - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, @@ -390,16 +387,14 @@ class WorkflowAppGenerator(BaseAppGenerator): # Create session factory session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) # Create workflow execution(aka workflow run) repository - workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository( + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, ) # Create workflow node execution repository - session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) - - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=user, app_id=application_generate_entity.app_config.app_id, diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 2a85cd5e3d..c6b326d8a4 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -3,7 +3,6 @@ import time from collections.abc import Generator from typing import Optional, Union -from sqlalchemy import select from sqlalchemy.orm import Session from constants.tts_auto_play_timeout import TTS_AUTO_PLAY_TIMEOUT, TTS_AUTO_PLAY_YIELD_CPU_TIME @@ -68,7 +67,6 @@ from models.workflow import ( Workflow, WorkflowAppLog, WorkflowAppLogCreatedFrom, - WorkflowRun, ) logger = logging.getLogger(__name__) @@ -562,8 +560,6 @@ class WorkflowAppGenerateTaskPipeline: tts_publisher.publish(None) def _save_workflow_app_log(self, *, session: Session, workflow_execution: WorkflowExecution) -> None: - workflow_run = session.scalar(select(WorkflowRun).where(WorkflowRun.id == workflow_execution.id_)) - assert workflow_run is not None invoke_from = self._application_generate_entity.invoke_from if invoke_from == InvokeFrom.SERVICE_API: created_from = WorkflowAppLogCreatedFrom.SERVICE_API @@ -576,10 +572,10 @@ class WorkflowAppGenerateTaskPipeline: return workflow_app_log = WorkflowAppLog() - workflow_app_log.tenant_id = workflow_run.tenant_id - workflow_app_log.app_id = workflow_run.app_id - workflow_app_log.workflow_id = workflow_run.workflow_id - workflow_app_log.workflow_run_id = workflow_run.id + workflow_app_log.tenant_id = self._application_generate_entity.app_config.tenant_id + workflow_app_log.app_id = self._application_generate_entity.app_config.app_id + workflow_app_log.workflow_id = workflow_execution.workflow_id + workflow_app_log.workflow_run_id = workflow_execution.id_ workflow_app_log.created_from = created_from.value workflow_app_log.created_by_role = self._created_by_role workflow_app_log.created_by = self._user_id diff --git a/api/core/entities/parameter_entities.py b/api/core/entities/parameter_entities.py index b071bfa5b1..2fa347c204 100644 --- a/api/core/entities/parameter_entities.py +++ b/api/core/entities/parameter_entities.py @@ -21,6 +21,9 @@ class CommonParameterType(StrEnum): DYNAMIC_SELECT = "dynamic-select" # TOOL_SELECTOR = "tool-selector" + # MCP object and array type parameters + ARRAY = "array" + OBJECT = "object" class AppSelectorScope(StrEnum): diff --git a/api/core/file/helpers.py b/api/core/file/helpers.py index 73fabdb11b..335ad2266a 100644 --- a/api/core/file/helpers.py +++ b/api/core/file/helpers.py @@ -21,7 +21,9 @@ def get_signed_file_url(upload_file_id: str) -> str: def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str, user_id: str) -> str: - url = f"{dify_config.FILES_URL}/files/upload/for-plugin" + # Plugin access should use internal URL for Docker network communication + base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL + url = f"{base_url}/files/upload/for-plugin" if user_id is None: user_id = "DEFAULT-USER" diff --git a/api/core/helper/code_executor/template_transformer.py b/api/core/helper/code_executor/template_transformer.py index 84f212a9c1..b416e48ce4 100644 --- a/api/core/helper/code_executor/template_transformer.py +++ b/api/core/helper/code_executor/template_transformer.py @@ -5,6 +5,8 @@ from base64 import b64encode from collections.abc import Mapping from typing import Any +from core.variables.utils import SegmentJSONEncoder + class TemplateTransformer(ABC): _code_placeholder: str = "{{code}}" @@ -43,17 +45,13 @@ class TemplateTransformer(ABC): result_str = cls.extract_result_str_from_response(response) result = json.loads(result_str) except json.JSONDecodeError as e: - raise ValueError(f"Failed to parse JSON response: {str(e)}. Response content: {result_str[:200]}...") + raise ValueError(f"Failed to parse JSON response: {str(e)}.") except ValueError as e: # Re-raise ValueError from extract_result_str_from_response raise e except Exception as e: raise ValueError(f"Unexpected error during response transformation: {str(e)}") - # Check if the result contains an error - if isinstance(result, dict) and "error" in result: - raise ValueError(f"JavaScript execution error: {result['error']}") - if not isinstance(result, dict): raise ValueError(f"Result must be a dict, got {type(result).__name__}") if not all(isinstance(k, str) for k in result): @@ -95,7 +93,7 @@ class TemplateTransformer(ABC): @classmethod def serialize_inputs(cls, inputs: Mapping[str, Any]) -> str: - inputs_json_str = json.dumps(inputs, ensure_ascii=False).encode() + inputs_json_str = json.dumps(inputs, ensure_ascii=False, cls=SegmentJSONEncoder).encode() input_base64_encoded = b64encode(inputs_json_str).decode("utf-8") return input_base64_encoded diff --git a/api/core/mcp/__init__.py b/api/core/mcp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/mcp/auth/auth_flow.py b/api/core/mcp/auth/auth_flow.py new file mode 100644 index 0000000000..b63478e822 --- /dev/null +++ b/api/core/mcp/auth/auth_flow.py @@ -0,0 +1,342 @@ +import base64 +import hashlib +import json +import os +import secrets +import urllib.parse +from typing import Optional +from urllib.parse import urljoin + +import requests +from pydantic import BaseModel, ValidationError + +from core.mcp.auth.auth_provider import OAuthClientProvider +from core.mcp.types import ( + OAuthClientInformation, + OAuthClientInformationFull, + OAuthClientMetadata, + OAuthMetadata, + OAuthTokens, +) +from extensions.ext_redis import redis_client + +LATEST_PROTOCOL_VERSION = "1.0" +OAUTH_STATE_EXPIRY_SECONDS = 5 * 60 # 5 minutes expiry +OAUTH_STATE_REDIS_KEY_PREFIX = "oauth_state:" + + +class OAuthCallbackState(BaseModel): + provider_id: str + tenant_id: str + server_url: str + metadata: OAuthMetadata | None = None + client_information: OAuthClientInformation + code_verifier: str + redirect_uri: str + + +def generate_pkce_challenge() -> tuple[str, str]: + """Generate PKCE challenge and verifier.""" + code_verifier = base64.urlsafe_b64encode(os.urandom(40)).decode("utf-8") + code_verifier = code_verifier.replace("=", "").replace("+", "-").replace("/", "_") + + code_challenge_hash = hashlib.sha256(code_verifier.encode("utf-8")).digest() + code_challenge = base64.urlsafe_b64encode(code_challenge_hash).decode("utf-8") + code_challenge = code_challenge.replace("=", "").replace("+", "-").replace("/", "_") + + return code_verifier, code_challenge + + +def _create_secure_redis_state(state_data: OAuthCallbackState) -> str: + """Create a secure state parameter by storing state data in Redis and returning a random state key.""" + # Generate a secure random state key + state_key = secrets.token_urlsafe(32) + + # Store the state data in Redis with expiration + redis_key = f"{OAUTH_STATE_REDIS_KEY_PREFIX}{state_key}" + redis_client.setex(redis_key, OAUTH_STATE_EXPIRY_SECONDS, state_data.model_dump_json()) + + return state_key + + +def _retrieve_redis_state(state_key: str) -> OAuthCallbackState: + """Retrieve and decode OAuth state data from Redis using the state key, then delete it.""" + redis_key = f"{OAUTH_STATE_REDIS_KEY_PREFIX}{state_key}" + + # Get state data from Redis + state_data = redis_client.get(redis_key) + + if not state_data: + raise ValueError("State parameter has expired or does not exist") + + # Delete the state data from Redis immediately after retrieval to prevent reuse + redis_client.delete(redis_key) + + try: + # Parse and validate the state data + oauth_state = OAuthCallbackState.model_validate_json(state_data) + + return oauth_state + except ValidationError as e: + raise ValueError(f"Invalid state parameter: {str(e)}") + + +def handle_callback(state_key: str, authorization_code: str) -> OAuthCallbackState: + """Handle the callback from the OAuth provider.""" + # Retrieve state data from Redis (state is automatically deleted after retrieval) + full_state_data = _retrieve_redis_state(state_key) + + tokens = exchange_authorization( + full_state_data.server_url, + full_state_data.metadata, + full_state_data.client_information, + authorization_code, + full_state_data.code_verifier, + full_state_data.redirect_uri, + ) + provider = OAuthClientProvider(full_state_data.provider_id, full_state_data.tenant_id, for_list=True) + provider.save_tokens(tokens) + return full_state_data + + +def discover_oauth_metadata(server_url: str, protocol_version: Optional[str] = None) -> Optional[OAuthMetadata]: + """Looks up RFC 8414 OAuth 2.0 Authorization Server Metadata.""" + url = urljoin(server_url, "/.well-known/oauth-authorization-server") + + try: + headers = {"MCP-Protocol-Version": protocol_version or LATEST_PROTOCOL_VERSION} + response = requests.get(url, headers=headers) + if response.status_code == 404: + return None + if not response.ok: + raise ValueError(f"HTTP {response.status_code} trying to load well-known OAuth metadata") + return OAuthMetadata.model_validate(response.json()) + except requests.RequestException as e: + if isinstance(e, requests.ConnectionError): + response = requests.get(url) + if response.status_code == 404: + return None + if not response.ok: + raise ValueError(f"HTTP {response.status_code} trying to load well-known OAuth metadata") + return OAuthMetadata.model_validate(response.json()) + raise + + +def start_authorization( + server_url: str, + metadata: Optional[OAuthMetadata], + client_information: OAuthClientInformation, + redirect_url: str, + provider_id: str, + tenant_id: str, +) -> tuple[str, str]: + """Begins the authorization flow with secure Redis state storage.""" + response_type = "code" + code_challenge_method = "S256" + + if metadata: + authorization_url = metadata.authorization_endpoint + if response_type not in metadata.response_types_supported: + raise ValueError(f"Incompatible auth server: does not support response type {response_type}") + if ( + not metadata.code_challenge_methods_supported + or code_challenge_method not in metadata.code_challenge_methods_supported + ): + raise ValueError( + f"Incompatible auth server: does not support code challenge method {code_challenge_method}" + ) + else: + authorization_url = urljoin(server_url, "/authorize") + + code_verifier, code_challenge = generate_pkce_challenge() + + # Prepare state data with all necessary information + state_data = OAuthCallbackState( + provider_id=provider_id, + tenant_id=tenant_id, + server_url=server_url, + metadata=metadata, + client_information=client_information, + code_verifier=code_verifier, + redirect_uri=redirect_url, + ) + + # Store state data in Redis and generate secure state key + state_key = _create_secure_redis_state(state_data) + + params = { + "response_type": response_type, + "client_id": client_information.client_id, + "code_challenge": code_challenge, + "code_challenge_method": code_challenge_method, + "redirect_uri": redirect_url, + "state": state_key, + } + + authorization_url = f"{authorization_url}?{urllib.parse.urlencode(params)}" + return authorization_url, code_verifier + + +def exchange_authorization( + server_url: str, + metadata: Optional[OAuthMetadata], + client_information: OAuthClientInformation, + authorization_code: str, + code_verifier: str, + redirect_uri: str, +) -> OAuthTokens: + """Exchanges an authorization code for an access token.""" + grant_type = "authorization_code" + + if metadata: + token_url = metadata.token_endpoint + if metadata.grant_types_supported and grant_type not in metadata.grant_types_supported: + raise ValueError(f"Incompatible auth server: does not support grant type {grant_type}") + else: + token_url = urljoin(server_url, "/token") + + params = { + "grant_type": grant_type, + "client_id": client_information.client_id, + "code": authorization_code, + "code_verifier": code_verifier, + "redirect_uri": redirect_uri, + } + + if client_information.client_secret: + params["client_secret"] = client_information.client_secret + + response = requests.post(token_url, data=params) + if not response.ok: + raise ValueError(f"Token exchange failed: HTTP {response.status_code}") + return OAuthTokens.model_validate(response.json()) + + +def refresh_authorization( + server_url: str, + metadata: Optional[OAuthMetadata], + client_information: OAuthClientInformation, + refresh_token: str, +) -> OAuthTokens: + """Exchange a refresh token for an updated access token.""" + grant_type = "refresh_token" + + if metadata: + token_url = metadata.token_endpoint + if metadata.grant_types_supported and grant_type not in metadata.grant_types_supported: + raise ValueError(f"Incompatible auth server: does not support grant type {grant_type}") + else: + token_url = urljoin(server_url, "/token") + + params = { + "grant_type": grant_type, + "client_id": client_information.client_id, + "refresh_token": refresh_token, + } + + if client_information.client_secret: + params["client_secret"] = client_information.client_secret + + response = requests.post(token_url, data=params) + if not response.ok: + raise ValueError(f"Token refresh failed: HTTP {response.status_code}") + return OAuthTokens.parse_obj(response.json()) + + +def register_client( + server_url: str, + metadata: Optional[OAuthMetadata], + client_metadata: OAuthClientMetadata, +) -> OAuthClientInformationFull: + """Performs OAuth 2.0 Dynamic Client Registration.""" + if metadata: + if not metadata.registration_endpoint: + raise ValueError("Incompatible auth server: does not support dynamic client registration") + registration_url = metadata.registration_endpoint + else: + registration_url = urljoin(server_url, "/register") + + response = requests.post( + registration_url, + json=client_metadata.model_dump(), + headers={"Content-Type": "application/json"}, + ) + if not response.ok: + response.raise_for_status() + return OAuthClientInformationFull.model_validate(response.json()) + + +def auth( + provider: OAuthClientProvider, + server_url: str, + authorization_code: Optional[str] = None, + state_param: Optional[str] = None, + for_list: bool = False, +) -> dict[str, str]: + """Orchestrates the full auth flow with a server using secure Redis state storage.""" + metadata = discover_oauth_metadata(server_url) + + # Handle client registration if needed + client_information = provider.client_information() + if not client_information: + if authorization_code is not None: + raise ValueError("Existing OAuth client information is required when exchanging an authorization code") + try: + full_information = register_client(server_url, metadata, provider.client_metadata) + except requests.RequestException as e: + raise ValueError(f"Could not register OAuth client: {e}") + provider.save_client_information(full_information) + client_information = full_information + + # Exchange authorization code for tokens + if authorization_code is not None: + if not state_param: + raise ValueError("State parameter is required when exchanging authorization code") + + try: + # Retrieve state data from Redis using state key + full_state_data = _retrieve_redis_state(state_param) + + code_verifier = full_state_data.code_verifier + redirect_uri = full_state_data.redirect_uri + + if not code_verifier or not redirect_uri: + raise ValueError("Missing code_verifier or redirect_uri in state data") + + except (json.JSONDecodeError, ValueError) as e: + raise ValueError(f"Invalid state parameter: {e}") + + tokens = exchange_authorization( + server_url, + metadata, + client_information, + authorization_code, + code_verifier, + redirect_uri, + ) + provider.save_tokens(tokens) + return {"result": "success"} + + provider_tokens = provider.tokens() + + # Handle token refresh or new authorization + if provider_tokens and provider_tokens.refresh_token: + try: + new_tokens = refresh_authorization(server_url, metadata, client_information, provider_tokens.refresh_token) + provider.save_tokens(new_tokens) + return {"result": "success"} + except Exception as e: + raise ValueError(f"Could not refresh OAuth tokens: {e}") + + # Start new authorization flow + authorization_url, code_verifier = start_authorization( + server_url, + metadata, + client_information, + provider.redirect_url, + provider.mcp_provider.id, + provider.mcp_provider.tenant_id, + ) + + provider.save_code_verifier(code_verifier) + return {"authorization_url": authorization_url} diff --git a/api/core/mcp/auth/auth_provider.py b/api/core/mcp/auth/auth_provider.py new file mode 100644 index 0000000000..cd55dbf64f --- /dev/null +++ b/api/core/mcp/auth/auth_provider.py @@ -0,0 +1,81 @@ +from typing import Optional + +from configs import dify_config +from core.mcp.types import ( + OAuthClientInformation, + OAuthClientInformationFull, + OAuthClientMetadata, + OAuthTokens, +) +from models.tools import MCPToolProvider +from services.tools.mcp_tools_mange_service import MCPToolManageService + +LATEST_PROTOCOL_VERSION = "1.0" + + +class OAuthClientProvider: + mcp_provider: MCPToolProvider + + def __init__(self, provider_id: str, tenant_id: str, for_list: bool = False): + if for_list: + self.mcp_provider = MCPToolManageService.get_mcp_provider_by_provider_id(provider_id, tenant_id) + else: + self.mcp_provider = MCPToolManageService.get_mcp_provider_by_server_identifier(provider_id, tenant_id) + + @property + def redirect_url(self) -> str: + """The URL to redirect the user agent to after authorization.""" + return dify_config.CONSOLE_API_URL + "/console/api/mcp/oauth/callback" + + @property + def client_metadata(self) -> OAuthClientMetadata: + """Metadata about this OAuth client.""" + return OAuthClientMetadata( + redirect_uris=[self.redirect_url], + token_endpoint_auth_method="none", + grant_types=["authorization_code", "refresh_token"], + response_types=["code"], + client_name="Dify", + client_uri="https://github.com/langgenius/dify", + ) + + def client_information(self) -> Optional[OAuthClientInformation]: + """Loads information about this OAuth client.""" + client_information = self.mcp_provider.decrypted_credentials.get("client_information", {}) + if not client_information: + return None + return OAuthClientInformation.model_validate(client_information) + + def save_client_information(self, client_information: OAuthClientInformationFull) -> None: + """Saves client information after dynamic registration.""" + MCPToolManageService.update_mcp_provider_credentials( + self.mcp_provider, + {"client_information": client_information.model_dump()}, + ) + + def tokens(self) -> Optional[OAuthTokens]: + """Loads any existing OAuth tokens for the current session.""" + credentials = self.mcp_provider.decrypted_credentials + if not credentials: + return None + return OAuthTokens( + access_token=credentials.get("access_token", ""), + token_type=credentials.get("token_type", "Bearer"), + expires_in=int(credentials.get("expires_in", "3600") or 3600), + refresh_token=credentials.get("refresh_token", ""), + ) + + def save_tokens(self, tokens: OAuthTokens) -> None: + """Stores new OAuth tokens for the current session.""" + # update mcp provider credentials + token_dict = tokens.model_dump() + MCPToolManageService.update_mcp_provider_credentials(self.mcp_provider, token_dict, authed=True) + + def save_code_verifier(self, code_verifier: str) -> None: + """Saves a PKCE code verifier for the current session.""" + MCPToolManageService.update_mcp_provider_credentials(self.mcp_provider, {"code_verifier": code_verifier}) + + def code_verifier(self) -> str: + """Loads the PKCE code verifier for the current session.""" + # get code verifier from mcp provider credentials + return str(self.mcp_provider.decrypted_credentials.get("code_verifier", "")) diff --git a/api/core/mcp/client/sse_client.py b/api/core/mcp/client/sse_client.py new file mode 100644 index 0000000000..91debcc8f9 --- /dev/null +++ b/api/core/mcp/client/sse_client.py @@ -0,0 +1,361 @@ +import logging +import queue +from collections.abc import Generator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from typing import Any, TypeAlias, final +from urllib.parse import urljoin, urlparse + +import httpx +from sseclient import SSEClient + +from core.mcp import types +from core.mcp.error import MCPAuthError, MCPConnectionError +from core.mcp.types import SessionMessage +from core.mcp.utils import create_ssrf_proxy_mcp_http_client, ssrf_proxy_sse_connect + +logger = logging.getLogger(__name__) + +DEFAULT_QUEUE_READ_TIMEOUT = 3 + + +@final +class _StatusReady: + def __init__(self, endpoint_url: str): + self._endpoint_url = endpoint_url + + +@final +class _StatusError: + def __init__(self, exc: Exception): + self._exc = exc + + +# Type aliases for better readability +ReadQueue: TypeAlias = queue.Queue[SessionMessage | Exception | None] +WriteQueue: TypeAlias = queue.Queue[SessionMessage | Exception | None] +StatusQueue: TypeAlias = queue.Queue[_StatusReady | _StatusError] + + +def remove_request_params(url: str) -> str: + """Remove request parameters from URL, keeping only the path.""" + return urljoin(url, urlparse(url).path) + + +class SSETransport: + """SSE client transport implementation.""" + + def __init__( + self, + url: str, + headers: dict[str, Any] | None = None, + timeout: float = 5.0, + sse_read_timeout: float = 5 * 60, + ) -> None: + """Initialize the SSE transport. + + Args: + url: The SSE endpoint URL. + headers: Optional headers to include in requests. + timeout: HTTP timeout for regular operations. + sse_read_timeout: Timeout for SSE read operations. + """ + self.url = url + self.headers = headers or {} + self.timeout = timeout + self.sse_read_timeout = sse_read_timeout + self.endpoint_url: str | None = None + + def _validate_endpoint_url(self, endpoint_url: str) -> bool: + """Validate that the endpoint URL matches the connection origin. + + Args: + endpoint_url: The endpoint URL to validate. + + Returns: + True if valid, False otherwise. + """ + url_parsed = urlparse(self.url) + endpoint_parsed = urlparse(endpoint_url) + + return url_parsed.netloc == endpoint_parsed.netloc and url_parsed.scheme == endpoint_parsed.scheme + + def _handle_endpoint_event(self, sse_data: str, status_queue: StatusQueue) -> None: + """Handle an 'endpoint' SSE event. + + Args: + sse_data: The SSE event data. + status_queue: Queue to put status updates. + """ + endpoint_url = urljoin(self.url, sse_data) + logger.info(f"Received endpoint URL: {endpoint_url}") + + if not self._validate_endpoint_url(endpoint_url): + error_msg = f"Endpoint origin does not match connection origin: {endpoint_url}" + logger.error(error_msg) + status_queue.put(_StatusError(ValueError(error_msg))) + return + + status_queue.put(_StatusReady(endpoint_url)) + + def _handle_message_event(self, sse_data: str, read_queue: ReadQueue) -> None: + """Handle a 'message' SSE event. + + Args: + sse_data: The SSE event data. + read_queue: Queue to put parsed messages. + """ + try: + message = types.JSONRPCMessage.model_validate_json(sse_data) + logger.debug(f"Received server message: {message}") + session_message = SessionMessage(message) + read_queue.put(session_message) + except Exception as exc: + logger.exception("Error parsing server message") + read_queue.put(exc) + + def _handle_sse_event(self, sse, read_queue: ReadQueue, status_queue: StatusQueue) -> None: + """Handle a single SSE event. + + Args: + sse: The SSE event object. + read_queue: Queue for message events. + status_queue: Queue for status events. + """ + match sse.event: + case "endpoint": + self._handle_endpoint_event(sse.data, status_queue) + case "message": + self._handle_message_event(sse.data, read_queue) + case _: + logger.warning(f"Unknown SSE event: {sse.event}") + + def sse_reader(self, event_source, read_queue: ReadQueue, status_queue: StatusQueue) -> None: + """Read and process SSE events. + + Args: + event_source: The SSE event source. + read_queue: Queue to put received messages. + status_queue: Queue to put status updates. + """ + try: + for sse in event_source.iter_sse(): + self._handle_sse_event(sse, read_queue, status_queue) + except httpx.ReadError as exc: + logger.debug(f"SSE reader shutting down normally: {exc}") + except Exception as exc: + read_queue.put(exc) + finally: + read_queue.put(None) + + def _send_message(self, client: httpx.Client, endpoint_url: str, message: SessionMessage) -> None: + """Send a single message to the server. + + Args: + client: HTTP client to use. + endpoint_url: The endpoint URL to send to. + message: The message to send. + """ + response = client.post( + endpoint_url, + json=message.message.model_dump( + by_alias=True, + mode="json", + exclude_none=True, + ), + ) + response.raise_for_status() + logger.debug(f"Client message sent successfully: {response.status_code}") + + def post_writer(self, client: httpx.Client, endpoint_url: str, write_queue: WriteQueue) -> None: + """Handle writing messages to the server. + + Args: + client: HTTP client to use. + endpoint_url: The endpoint URL to send messages to. + write_queue: Queue to read messages from. + """ + try: + while True: + try: + message = write_queue.get(timeout=DEFAULT_QUEUE_READ_TIMEOUT) + if message is None: + break + if isinstance(message, Exception): + write_queue.put(message) + continue + + self._send_message(client, endpoint_url, message) + + except queue.Empty: + continue + except httpx.ReadError as exc: + logger.debug(f"Post writer shutting down normally: {exc}") + except Exception as exc: + logger.exception("Error writing messages") + write_queue.put(exc) + finally: + write_queue.put(None) + + def _wait_for_endpoint(self, status_queue: StatusQueue) -> str: + """Wait for the endpoint URL from the status queue. + + Args: + status_queue: Queue to read status from. + + Returns: + The endpoint URL. + + Raises: + ValueError: If endpoint URL is not received or there's an error. + """ + try: + status = status_queue.get(timeout=1) + except queue.Empty: + raise ValueError("failed to get endpoint URL") + + if isinstance(status, _StatusReady): + return status._endpoint_url + elif isinstance(status, _StatusError): + raise status._exc + else: + raise ValueError("failed to get endpoint URL") + + def connect( + self, + executor: ThreadPoolExecutor, + client: httpx.Client, + event_source, + ) -> tuple[ReadQueue, WriteQueue]: + """Establish connection and start worker threads. + + Args: + executor: Thread pool executor. + client: HTTP client. + event_source: SSE event source. + + Returns: + Tuple of (read_queue, write_queue). + """ + read_queue: ReadQueue = queue.Queue() + write_queue: WriteQueue = queue.Queue() + status_queue: StatusQueue = queue.Queue() + + # Start SSE reader thread + executor.submit(self.sse_reader, event_source, read_queue, status_queue) + + # Wait for endpoint URL + endpoint_url = self._wait_for_endpoint(status_queue) + self.endpoint_url = endpoint_url + + # Start post writer thread + executor.submit(self.post_writer, client, endpoint_url, write_queue) + + return read_queue, write_queue + + +@contextmanager +def sse_client( + url: str, + headers: dict[str, Any] | None = None, + timeout: float = 5.0, + sse_read_timeout: float = 5 * 60, +) -> Generator[tuple[ReadQueue, WriteQueue], None, None]: + """ + Client transport for SSE. + `sse_read_timeout` determines how long (in seconds) the client will wait for a new + event before disconnecting. All other HTTP operations are controlled by `timeout`. + + Args: + url: The SSE endpoint URL. + headers: Optional headers to include in requests. + timeout: HTTP timeout for regular operations. + sse_read_timeout: Timeout for SSE read operations. + + Yields: + Tuple of (read_queue, write_queue) for message communication. + """ + transport = SSETransport(url, headers, timeout, sse_read_timeout) + + read_queue: ReadQueue | None = None + write_queue: WriteQueue | None = None + + with ThreadPoolExecutor() as executor: + try: + with create_ssrf_proxy_mcp_http_client(headers=transport.headers) as client: + with ssrf_proxy_sse_connect( + url, timeout=httpx.Timeout(timeout, read=sse_read_timeout), client=client + ) as event_source: + event_source.response.raise_for_status() + + read_queue, write_queue = transport.connect(executor, client, event_source) + + yield read_queue, write_queue + + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 401: + raise MCPAuthError() + raise MCPConnectionError() + except Exception: + logger.exception("Error connecting to SSE endpoint") + raise + finally: + # Clean up queues + if read_queue: + read_queue.put(None) + if write_queue: + write_queue.put(None) + + +def send_message(http_client: httpx.Client, endpoint_url: str, session_message: SessionMessage) -> None: + """ + Send a message to the server using the provided HTTP client. + + Args: + http_client: The HTTP client to use for sending + endpoint_url: The endpoint URL to send the message to + session_message: The message to send + """ + try: + response = http_client.post( + endpoint_url, + json=session_message.message.model_dump( + by_alias=True, + mode="json", + exclude_none=True, + ), + ) + response.raise_for_status() + logger.debug(f"Client message sent successfully: {response.status_code}") + except Exception as exc: + logger.exception("Error sending message") + raise + + +def read_messages( + sse_client: SSEClient, +) -> Generator[SessionMessage | Exception, None, None]: + """ + Read messages from the SSE client. + + Args: + sse_client: The SSE client to read from + + Yields: + SessionMessage or Exception for each event received + """ + try: + for sse in sse_client.events(): + if sse.event == "message": + try: + message = types.JSONRPCMessage.model_validate_json(sse.data) + logger.debug(f"Received server message: {message}") + yield SessionMessage(message) + except Exception as exc: + logger.exception("Error parsing server message") + yield exc + else: + logger.warning(f"Unknown SSE event: {sse.event}") + except Exception as exc: + logger.exception("Error reading SSE messages") + yield exc diff --git a/api/core/mcp/client/streamable_client.py b/api/core/mcp/client/streamable_client.py new file mode 100644 index 0000000000..fbd8d05f9e --- /dev/null +++ b/api/core/mcp/client/streamable_client.py @@ -0,0 +1,476 @@ +""" +StreamableHTTP Client Transport Module + +This module implements the StreamableHTTP transport for MCP clients, +providing support for HTTP POST requests with optional SSE streaming responses +and session management. +""" + +import logging +import queue +from collections.abc import Callable, Generator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, cast + +import httpx +from httpx_sse import EventSource, ServerSentEvent + +from core.mcp.types import ( + ClientMessageMetadata, + ErrorData, + JSONRPCError, + JSONRPCMessage, + JSONRPCNotification, + JSONRPCRequest, + JSONRPCResponse, + RequestId, + SessionMessage, +) +from core.mcp.utils import create_ssrf_proxy_mcp_http_client, ssrf_proxy_sse_connect + +logger = logging.getLogger(__name__) + + +SessionMessageOrError = SessionMessage | Exception | None +# Queue types with clearer names for their roles +ServerToClientQueue = queue.Queue[SessionMessageOrError] # Server to client messages +ClientToServerQueue = queue.Queue[SessionMessage | None] # Client to server messages +GetSessionIdCallback = Callable[[], str | None] + +MCP_SESSION_ID = "mcp-session-id" +LAST_EVENT_ID = "last-event-id" +CONTENT_TYPE = "content-type" +ACCEPT = "Accept" + + +JSON = "application/json" +SSE = "text/event-stream" + +DEFAULT_QUEUE_READ_TIMEOUT = 3 + + +class StreamableHTTPError(Exception): + """Base exception for StreamableHTTP transport errors.""" + + pass + + +class ResumptionError(StreamableHTTPError): + """Raised when resumption request is invalid.""" + + pass + + +@dataclass +class RequestContext: + """Context for a request operation.""" + + client: httpx.Client + headers: dict[str, str] + session_id: str | None + session_message: SessionMessage + metadata: ClientMessageMetadata | None + server_to_client_queue: ServerToClientQueue # Renamed for clarity + sse_read_timeout: timedelta + + +class StreamableHTTPTransport: + """StreamableHTTP client transport implementation.""" + + def __init__( + self, + url: str, + headers: dict[str, Any] | None = None, + timeout: timedelta = timedelta(seconds=30), + sse_read_timeout: timedelta = timedelta(seconds=60 * 5), + ) -> None: + """Initialize the StreamableHTTP transport. + + Args: + url: The endpoint URL. + headers: Optional headers to include in requests. + timeout: HTTP timeout for regular operations. + sse_read_timeout: Timeout for SSE read operations. + """ + self.url = url + self.headers = headers or {} + self.timeout = timeout + self.sse_read_timeout = sse_read_timeout + self.session_id: str | None = None + self.request_headers = { + ACCEPT: f"{JSON}, {SSE}", + CONTENT_TYPE: JSON, + **self.headers, + } + + def _update_headers_with_session(self, base_headers: dict[str, str]) -> dict[str, str]: + """Update headers with session ID if available.""" + headers = base_headers.copy() + if self.session_id: + headers[MCP_SESSION_ID] = self.session_id + return headers + + def _is_initialization_request(self, message: JSONRPCMessage) -> bool: + """Check if the message is an initialization request.""" + return isinstance(message.root, JSONRPCRequest) and message.root.method == "initialize" + + def _is_initialized_notification(self, message: JSONRPCMessage) -> bool: + """Check if the message is an initialized notification.""" + return isinstance(message.root, JSONRPCNotification) and message.root.method == "notifications/initialized" + + def _maybe_extract_session_id_from_response( + self, + response: httpx.Response, + ) -> None: + """Extract and store session ID from response headers.""" + new_session_id = response.headers.get(MCP_SESSION_ID) + if new_session_id: + self.session_id = new_session_id + logger.info(f"Received session ID: {self.session_id}") + + def _handle_sse_event( + self, + sse: ServerSentEvent, + server_to_client_queue: ServerToClientQueue, + original_request_id: RequestId | None = None, + resumption_callback: Callable[[str], None] | None = None, + ) -> bool: + """Handle an SSE event, returning True if the response is complete.""" + if sse.event == "message": + try: + message = JSONRPCMessage.model_validate_json(sse.data) + logger.debug(f"SSE message: {message}") + + # If this is a response and we have original_request_id, replace it + if original_request_id is not None and isinstance(message.root, JSONRPCResponse | JSONRPCError): + message.root.id = original_request_id + + session_message = SessionMessage(message) + # Put message in queue that goes to client + server_to_client_queue.put(session_message) + + # Call resumption token callback if we have an ID + if sse.id and resumption_callback: + resumption_callback(sse.id) + + # If this is a response or error return True indicating completion + # Otherwise, return False to continue listening + return isinstance(message.root, JSONRPCResponse | JSONRPCError) + + except Exception as exc: + # Put exception in queue that goes to client + server_to_client_queue.put(exc) + return False + elif sse.event == "ping": + logger.debug("Received ping event") + return False + else: + logger.warning(f"Unknown SSE event: {sse.event}") + return False + + def handle_get_stream( + self, + client: httpx.Client, + server_to_client_queue: ServerToClientQueue, + ) -> None: + """Handle GET stream for server-initiated messages.""" + try: + if not self.session_id: + return + + headers = self._update_headers_with_session(self.request_headers) + + with ssrf_proxy_sse_connect( + self.url, + headers=headers, + timeout=httpx.Timeout(self.timeout.seconds, read=self.sse_read_timeout.seconds), + client=client, + method="GET", + ) as event_source: + event_source.response.raise_for_status() + logger.debug("GET SSE connection established") + + for sse in event_source.iter_sse(): + self._handle_sse_event(sse, server_to_client_queue) + + except Exception as exc: + logger.debug(f"GET stream error (non-fatal): {exc}") + + def _handle_resumption_request(self, ctx: RequestContext) -> None: + """Handle a resumption request using GET with SSE.""" + headers = self._update_headers_with_session(ctx.headers) + if ctx.metadata and ctx.metadata.resumption_token: + headers[LAST_EVENT_ID] = ctx.metadata.resumption_token + else: + raise ResumptionError("Resumption request requires a resumption token") + + # Extract original request ID to map responses + original_request_id = None + if isinstance(ctx.session_message.message.root, JSONRPCRequest): + original_request_id = ctx.session_message.message.root.id + + with ssrf_proxy_sse_connect( + self.url, + headers=headers, + timeout=httpx.Timeout(self.timeout.seconds, read=ctx.sse_read_timeout.seconds), + client=ctx.client, + method="GET", + ) as event_source: + event_source.response.raise_for_status() + logger.debug("Resumption GET SSE connection established") + + for sse in event_source.iter_sse(): + is_complete = self._handle_sse_event( + sse, + ctx.server_to_client_queue, + original_request_id, + ctx.metadata.on_resumption_token_update if ctx.metadata else None, + ) + if is_complete: + break + + def _handle_post_request(self, ctx: RequestContext) -> None: + """Handle a POST request with response processing.""" + headers = self._update_headers_with_session(ctx.headers) + message = ctx.session_message.message + is_initialization = self._is_initialization_request(message) + + with ctx.client.stream( + "POST", + self.url, + json=message.model_dump(by_alias=True, mode="json", exclude_none=True), + headers=headers, + ) as response: + if response.status_code == 202: + logger.debug("Received 202 Accepted") + return + + if response.status_code == 404: + if isinstance(message.root, JSONRPCRequest): + self._send_session_terminated_error( + ctx.server_to_client_queue, + message.root.id, + ) + return + + response.raise_for_status() + if is_initialization: + self._maybe_extract_session_id_from_response(response) + + content_type = cast(str, response.headers.get(CONTENT_TYPE, "").lower()) + + if content_type.startswith(JSON): + self._handle_json_response(response, ctx.server_to_client_queue) + elif content_type.startswith(SSE): + self._handle_sse_response(response, ctx) + else: + self._handle_unexpected_content_type( + content_type, + ctx.server_to_client_queue, + ) + + def _handle_json_response( + self, + response: httpx.Response, + server_to_client_queue: ServerToClientQueue, + ) -> None: + """Handle JSON response from the server.""" + try: + content = response.read() + message = JSONRPCMessage.model_validate_json(content) + session_message = SessionMessage(message) + server_to_client_queue.put(session_message) + except Exception as exc: + server_to_client_queue.put(exc) + + def _handle_sse_response(self, response: httpx.Response, ctx: RequestContext) -> None: + """Handle SSE response from the server.""" + try: + event_source = EventSource(response) + for sse in event_source.iter_sse(): + is_complete = self._handle_sse_event( + sse, + ctx.server_to_client_queue, + resumption_callback=(ctx.metadata.on_resumption_token_update if ctx.metadata else None), + ) + if is_complete: + break + except Exception as e: + ctx.server_to_client_queue.put(e) + + def _handle_unexpected_content_type( + self, + content_type: str, + server_to_client_queue: ServerToClientQueue, + ) -> None: + """Handle unexpected content type in response.""" + error_msg = f"Unexpected content type: {content_type}" + logger.error(error_msg) + server_to_client_queue.put(ValueError(error_msg)) + + def _send_session_terminated_error( + self, + server_to_client_queue: ServerToClientQueue, + request_id: RequestId, + ) -> None: + """Send a session terminated error response.""" + jsonrpc_error = JSONRPCError( + jsonrpc="2.0", + id=request_id, + error=ErrorData(code=32600, message="Session terminated by server"), + ) + session_message = SessionMessage(JSONRPCMessage(jsonrpc_error)) + server_to_client_queue.put(session_message) + + def post_writer( + self, + client: httpx.Client, + client_to_server_queue: ClientToServerQueue, + server_to_client_queue: ServerToClientQueue, + start_get_stream: Callable[[], None], + ) -> None: + """Handle writing requests to the server. + + This method processes messages from the client_to_server_queue and sends them to the server. + Responses are written to the server_to_client_queue. + """ + while True: + try: + # Read message from client queue with timeout to check stop_event periodically + session_message = client_to_server_queue.get(timeout=DEFAULT_QUEUE_READ_TIMEOUT) + if session_message is None: + break + + message = session_message.message + metadata = ( + session_message.metadata if isinstance(session_message.metadata, ClientMessageMetadata) else None + ) + + # Check if this is a resumption request + is_resumption = bool(metadata and metadata.resumption_token) + + logger.debug(f"Sending client message: {message}") + + # Handle initialized notification + if self._is_initialized_notification(message): + start_get_stream() + + ctx = RequestContext( + client=client, + headers=self.request_headers, + session_id=self.session_id, + session_message=session_message, + metadata=metadata, + server_to_client_queue=server_to_client_queue, # Queue to write responses to client + sse_read_timeout=self.sse_read_timeout, + ) + + if is_resumption: + self._handle_resumption_request(ctx) + else: + self._handle_post_request(ctx) + except queue.Empty: + continue + except Exception as exc: + server_to_client_queue.put(exc) + + def terminate_session(self, client: httpx.Client) -> None: + """Terminate the session by sending a DELETE request.""" + if not self.session_id: + return + + try: + headers = self._update_headers_with_session(self.request_headers) + response = client.delete(self.url, headers=headers) + + if response.status_code == 405: + logger.debug("Server does not allow session termination") + elif response.status_code != 200: + logger.warning(f"Session termination failed: {response.status_code}") + except Exception as exc: + logger.warning(f"Session termination failed: {exc}") + + def get_session_id(self) -> str | None: + """Get the current session ID.""" + return self.session_id + + +@contextmanager +def streamablehttp_client( + url: str, + headers: dict[str, Any] | None = None, + timeout: timedelta = timedelta(seconds=30), + sse_read_timeout: timedelta = timedelta(seconds=60 * 5), + terminate_on_close: bool = True, +) -> Generator[ + tuple[ + ServerToClientQueue, # Queue for receiving messages FROM server + ClientToServerQueue, # Queue for sending messages TO server + GetSessionIdCallback, + ], + None, + None, +]: + """ + Client transport for StreamableHTTP. + + `sse_read_timeout` determines how long (in seconds) the client will wait for a new + event before disconnecting. All other HTTP operations are controlled by `timeout`. + + Yields: + Tuple containing: + - server_to_client_queue: Queue for reading messages FROM the server + - client_to_server_queue: Queue for sending messages TO the server + - get_session_id_callback: Function to retrieve the current session ID + """ + transport = StreamableHTTPTransport(url, headers, timeout, sse_read_timeout) + + # Create queues with clear directional meaning + server_to_client_queue: ServerToClientQueue = queue.Queue() # For messages FROM server TO client + client_to_server_queue: ClientToServerQueue = queue.Queue() # For messages FROM client TO server + + with ThreadPoolExecutor(max_workers=2) as executor: + try: + with create_ssrf_proxy_mcp_http_client( + headers=transport.request_headers, + timeout=httpx.Timeout(transport.timeout.seconds, read=transport.sse_read_timeout.seconds), + ) as client: + # Define callbacks that need access to thread pool + def start_get_stream() -> None: + """Start a worker thread to handle server-initiated messages.""" + executor.submit(transport.handle_get_stream, client, server_to_client_queue) + + # Start the post_writer worker thread + executor.submit( + transport.post_writer, + client, + client_to_server_queue, # Queue for messages FROM client TO server + server_to_client_queue, # Queue for messages FROM server TO client + start_get_stream, + ) + + try: + yield ( + server_to_client_queue, # Queue for receiving messages FROM server + client_to_server_queue, # Queue for sending messages TO server + transport.get_session_id, + ) + finally: + if transport.session_id and terminate_on_close: + transport.terminate_session(client) + + # Signal threads to stop + client_to_server_queue.put(None) + finally: + # Clear any remaining items and add None sentinel to unblock any waiting threads + try: + while not client_to_server_queue.empty(): + client_to_server_queue.get_nowait() + except queue.Empty: + pass + + client_to_server_queue.put(None) + server_to_client_queue.put(None) diff --git a/api/core/mcp/entities.py b/api/core/mcp/entities.py new file mode 100644 index 0000000000..7553c10a2e --- /dev/null +++ b/api/core/mcp/entities.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from typing import Any, Generic, TypeVar + +from core.mcp.session.base_session import BaseSession +from core.mcp.types import LATEST_PROTOCOL_VERSION, RequestId, RequestParams + +SUPPORTED_PROTOCOL_VERSIONS: list[str] = ["2024-11-05", LATEST_PROTOCOL_VERSION] + + +SessionT = TypeVar("SessionT", bound=BaseSession[Any, Any, Any, Any, Any]) +LifespanContextT = TypeVar("LifespanContextT") + + +@dataclass +class RequestContext(Generic[SessionT, LifespanContextT]): + request_id: RequestId + meta: RequestParams.Meta | None + session: SessionT + lifespan_context: LifespanContextT diff --git a/api/core/mcp/error.py b/api/core/mcp/error.py new file mode 100644 index 0000000000..92ea7bde09 --- /dev/null +++ b/api/core/mcp/error.py @@ -0,0 +1,10 @@ +class MCPError(Exception): + pass + + +class MCPConnectionError(MCPError): + pass + + +class MCPAuthError(MCPConnectionError): + pass diff --git a/api/core/mcp/mcp_client.py b/api/core/mcp/mcp_client.py new file mode 100644 index 0000000000..e9036de8c6 --- /dev/null +++ b/api/core/mcp/mcp_client.py @@ -0,0 +1,150 @@ +import logging +from collections.abc import Callable +from contextlib import AbstractContextManager, ExitStack +from types import TracebackType +from typing import Any, Optional, cast +from urllib.parse import urlparse + +from core.mcp.client.sse_client import sse_client +from core.mcp.client.streamable_client import streamablehttp_client +from core.mcp.error import MCPAuthError, MCPConnectionError +from core.mcp.session.client_session import ClientSession +from core.mcp.types import Tool + +logger = logging.getLogger(__name__) + + +class MCPClient: + def __init__( + self, + server_url: str, + provider_id: str, + tenant_id: str, + authed: bool = True, + authorization_code: Optional[str] = None, + for_list: bool = False, + ): + # Initialize info + self.provider_id = provider_id + self.tenant_id = tenant_id + self.client_type = "streamable" + self.server_url = server_url + + # Authentication info + self.authed = authed + self.authorization_code = authorization_code + if authed: + from core.mcp.auth.auth_provider import OAuthClientProvider + + self.provider = OAuthClientProvider(self.provider_id, self.tenant_id, for_list=for_list) + self.token = self.provider.tokens() + + # Initialize session and client objects + self._session: Optional[ClientSession] = None + self._streams_context: Optional[AbstractContextManager[Any]] = None + self._session_context: Optional[ClientSession] = None + self.exit_stack = ExitStack() + + # Whether the client has been initialized + self._initialized = False + + def __enter__(self): + self._initialize() + self._initialized = True + return self + + def __exit__( + self, exc_type: Optional[type], exc_value: Optional[BaseException], traceback: Optional[TracebackType] + ): + self.cleanup() + + def _initialize( + self, + ): + """Initialize the client with fallback to SSE if streamable connection fails""" + connection_methods: dict[str, Callable[..., AbstractContextManager[Any]]] = { + "mcp": streamablehttp_client, + "sse": sse_client, + } + + parsed_url = urlparse(self.server_url) + path = parsed_url.path + method_name = path.rstrip("/").split("/")[-1] if path else "" + try: + client_factory = connection_methods[method_name] + self.connect_server(client_factory, method_name) + except KeyError: + try: + self.connect_server(sse_client, "sse") + except MCPConnectionError: + self.connect_server(streamablehttp_client, "mcp") + + def connect_server( + self, client_factory: Callable[..., AbstractContextManager[Any]], method_name: str, first_try: bool = True + ): + from core.mcp.auth.auth_flow import auth + + try: + headers = ( + {"Authorization": f"{self.token.token_type.capitalize()} {self.token.access_token}"} + if self.authed and self.token + else {} + ) + self._streams_context = client_factory(url=self.server_url, headers=headers) + if self._streams_context is None: + raise MCPConnectionError("Failed to create connection context") + + # Use exit_stack to manage context managers properly + if method_name == "mcp": + read_stream, write_stream, _ = self.exit_stack.enter_context(self._streams_context) + streams = (read_stream, write_stream) + else: # sse_client + streams = self.exit_stack.enter_context(self._streams_context) + + self._session_context = ClientSession(*streams) + self._session = self.exit_stack.enter_context(self._session_context) + session = cast(ClientSession, self._session) + session.initialize() + return + + except MCPAuthError: + if not self.authed: + raise + try: + auth(self.provider, self.server_url, self.authorization_code) + except Exception as e: + raise ValueError(f"Failed to authenticate: {e}") + self.token = self.provider.tokens() + if first_try: + return self.connect_server(client_factory, method_name, first_try=False) + + except MCPConnectionError: + raise + + def list_tools(self) -> list[Tool]: + """Connect to an MCP server running with SSE transport""" + # List available tools to verify connection + if not self._initialized or not self._session: + raise ValueError("Session not initialized.") + response = self._session.list_tools() + tools = response.tools + return tools + + def invoke_tool(self, tool_name: str, tool_args: dict): + """Call a tool""" + if not self._initialized or not self._session: + raise ValueError("Session not initialized.") + return self._session.call_tool(tool_name, tool_args) + + def cleanup(self): + """Clean up resources""" + try: + # ExitStack will handle proper cleanup of all managed context managers + self.exit_stack.close() + self._session = None + self._session_context = None + self._streams_context = None + self._initialized = False + except Exception as e: + logging.exception("Error during cleanup") + raise ValueError(f"Error during cleanup: {e}") diff --git a/api/core/mcp/server/streamable_http.py b/api/core/mcp/server/streamable_http.py new file mode 100644 index 0000000000..1c2cf570e2 --- /dev/null +++ b/api/core/mcp/server/streamable_http.py @@ -0,0 +1,228 @@ +import json +import logging +from collections.abc import Mapping +from typing import Any, cast + +from configs import dify_config +from controllers.web.passport import generate_session_id +from core.app.app_config.entities import VariableEntity, VariableEntityType +from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.features.rate_limiting.rate_limit import RateLimitGenerator +from core.mcp import types +from core.mcp.types import INTERNAL_ERROR, INVALID_PARAMS, METHOD_NOT_FOUND +from core.mcp.utils import create_mcp_error_response +from core.model_runtime.utils.encoders import jsonable_encoder +from extensions.ext_database import db +from models.model import App, AppMCPServer, AppMode, EndUser +from services.app_generate_service import AppGenerateService + +""" +Apply to MCP HTTP streamable server with stateless http +""" +logger = logging.getLogger(__name__) + + +class MCPServerStreamableHTTPRequestHandler: + def __init__( + self, app: App, request: types.ClientRequest | types.ClientNotification, user_input_form: list[VariableEntity] + ): + self.app = app + self.request = request + mcp_server = db.session.query(AppMCPServer).filter(AppMCPServer.app_id == self.app.id).first() + if not mcp_server: + raise ValueError("MCP server not found") + self.mcp_server: AppMCPServer = mcp_server + self.end_user = self.retrieve_end_user() + self.user_input_form = user_input_form + + @property + def request_type(self): + return type(self.request.root) + + @property + def parameter_schema(self): + parameters, required = self._convert_input_form_to_parameters(self.user_input_form) + if self.app.mode in {AppMode.COMPLETION.value, AppMode.WORKFLOW.value}: + return { + "type": "object", + "properties": parameters, + "required": required, + } + return { + "type": "object", + "properties": { + "query": {"type": "string", "description": "User Input/Question content"}, + **parameters, + }, + "required": ["query", *required], + } + + @property + def capabilities(self): + return types.ServerCapabilities( + tools=types.ToolsCapability(listChanged=False), + ) + + def response(self, response: types.Result | str): + if isinstance(response, str): + sse_content = f"event: ping\ndata: {response}\n\n".encode() + yield sse_content + return + json_response = types.JSONRPCResponse( + jsonrpc="2.0", + id=(self.request.root.model_extra or {}).get("id", 1), + result=response.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + json_data = json.dumps(jsonable_encoder(json_response)) + + sse_content = f"event: message\ndata: {json_data}\n\n".encode() + + yield sse_content + + def error_response(self, code: int, message: str, data=None): + request_id = (self.request.root.model_extra or {}).get("id", 1) or 1 + return create_mcp_error_response(request_id, code, message, data) + + def handle(self): + handle_map = { + types.InitializeRequest: self.initialize, + types.ListToolsRequest: self.list_tools, + types.CallToolRequest: self.invoke_tool, + types.InitializedNotification: self.handle_notification, + types.PingRequest: self.handle_ping, + } + try: + if self.request_type in handle_map: + return self.response(handle_map[self.request_type]()) + else: + return self.error_response(METHOD_NOT_FOUND, f"Method not found: {self.request_type}") + except ValueError as e: + logger.exception("Invalid params") + return self.error_response(INVALID_PARAMS, str(e)) + except Exception as e: + logger.exception("Internal server error") + return self.error_response(INTERNAL_ERROR, f"Internal server error: {str(e)}") + + def handle_notification(self): + return "ping" + + def handle_ping(self): + return types.EmptyResult() + + def initialize(self): + request = cast(types.InitializeRequest, self.request.root) + client_info = request.params.clientInfo + client_name = f"{client_info.name}@{client_info.version}" + if not self.end_user: + end_user = EndUser( + tenant_id=self.app.tenant_id, + app_id=self.app.id, + type="mcp", + name=client_name, + session_id=generate_session_id(), + external_user_id=self.mcp_server.id, + ) + db.session.add(end_user) + db.session.commit() + return types.InitializeResult( + protocolVersion=types.SERVER_LATEST_PROTOCOL_VERSION, + capabilities=self.capabilities, + serverInfo=types.Implementation(name="Dify", version=dify_config.project.version), + instructions=self.mcp_server.description, + ) + + def list_tools(self): + if not self.end_user: + raise ValueError("User not found") + return types.ListToolsResult( + tools=[ + types.Tool( + name=self.app.name, + description=self.mcp_server.description, + inputSchema=self.parameter_schema, + ) + ], + ) + + def invoke_tool(self): + if not self.end_user: + raise ValueError("User not found") + request = cast(types.CallToolRequest, self.request.root) + args = request.params.arguments + if not args: + raise ValueError("No arguments provided") + if self.app.mode in {AppMode.WORKFLOW.value}: + args = {"inputs": args} + elif self.app.mode in {AppMode.COMPLETION.value}: + args = {"query": "", "inputs": args} + else: + args = {"query": args["query"], "inputs": {k: v for k, v in args.items() if k != "query"}} + response = AppGenerateService.generate( + self.app, + self.end_user, + args, + InvokeFrom.SERVICE_API, + streaming=self.app.mode == AppMode.AGENT_CHAT.value, + ) + answer = "" + if isinstance(response, RateLimitGenerator): + for item in response.generator: + data = item + if isinstance(data, str) and data.startswith("data: "): + try: + json_str = data[6:].strip() + parsed_data = json.loads(json_str) + if parsed_data.get("event") == "agent_thought": + answer += parsed_data.get("thought", "") + except json.JSONDecodeError: + continue + if isinstance(response, Mapping): + if self.app.mode in { + AppMode.ADVANCED_CHAT.value, + AppMode.COMPLETION.value, + AppMode.CHAT.value, + AppMode.AGENT_CHAT.value, + }: + answer = response["answer"] + elif self.app.mode in {AppMode.WORKFLOW.value}: + answer = json.dumps(response["data"]["outputs"], ensure_ascii=False) + else: + raise ValueError("Invalid app mode") + # Not support image yet + return types.CallToolResult(content=[types.TextContent(text=answer, type="text")]) + + def retrieve_end_user(self): + return ( + db.session.query(EndUser) + .filter(EndUser.external_user_id == self.mcp_server.id, EndUser.type == "mcp") + .first() + ) + + def _convert_input_form_to_parameters(self, user_input_form: list[VariableEntity]): + parameters: dict[str, dict[str, Any]] = {} + required = [] + for item in user_input_form: + parameters[item.variable] = {} + if item.type in ( + VariableEntityType.FILE, + VariableEntityType.FILE_LIST, + VariableEntityType.EXTERNAL_DATA_TOOL, + ): + continue + if item.required: + required.append(item.variable) + # if the workflow republished, the parameters not changed + # we should not raise error here + try: + description = self.mcp_server.parameters_dict[item.variable] + except KeyError: + description = "" + parameters[item.variable]["description"] = description + if item.type in (VariableEntityType.TEXT_INPUT, VariableEntityType.PARAGRAPH): + parameters[item.variable]["type"] = "string" + elif item.type == VariableEntityType.SELECT: + parameters[item.variable]["type"] = "string" + parameters[item.variable]["enum"] = item.options + elif item.type == VariableEntityType.NUMBER: + parameters[item.variable]["type"] = "float" + return parameters, required diff --git a/api/core/mcp/session/base_session.py b/api/core/mcp/session/base_session.py new file mode 100644 index 0000000000..1c0f582501 --- /dev/null +++ b/api/core/mcp/session/base_session.py @@ -0,0 +1,397 @@ +import logging +import queue +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from datetime import timedelta +from types import TracebackType +from typing import Any, Generic, Self, TypeVar + +from httpx import HTTPStatusError +from pydantic import BaseModel + +from core.mcp.error import MCPAuthError, MCPConnectionError +from core.mcp.types import ( + CancelledNotification, + ClientNotification, + ClientRequest, + ClientResult, + ErrorData, + JSONRPCError, + JSONRPCMessage, + JSONRPCNotification, + JSONRPCRequest, + JSONRPCResponse, + MessageMetadata, + RequestId, + RequestParams, + ServerMessageMetadata, + ServerNotification, + ServerRequest, + ServerResult, + SessionMessage, +) + +SendRequestT = TypeVar("SendRequestT", ClientRequest, ServerRequest) +SendResultT = TypeVar("SendResultT", ClientResult, ServerResult) +SendNotificationT = TypeVar("SendNotificationT", ClientNotification, ServerNotification) +ReceiveRequestT = TypeVar("ReceiveRequestT", ClientRequest, ServerRequest) +ReceiveResultT = TypeVar("ReceiveResultT", bound=BaseModel) +ReceiveNotificationT = TypeVar("ReceiveNotificationT", ClientNotification, ServerNotification) +DEFAULT_RESPONSE_READ_TIMEOUT = 1.0 + + +class RequestResponder(Generic[ReceiveRequestT, SendResultT]): + """Handles responding to MCP requests and manages request lifecycle. + + This class MUST be used as a context manager to ensure proper cleanup and + cancellation handling: + + Example: + with request_responder as resp: + resp.respond(result) + + The context manager ensures: + 1. Proper cancellation scope setup and cleanup + 2. Request completion tracking + 3. Cleanup of in-flight requests + """ + + request: ReceiveRequestT + _session: Any + _on_complete: Callable[["RequestResponder[ReceiveRequestT, SendResultT]"], Any] + + def __init__( + self, + request_id: RequestId, + request_meta: RequestParams.Meta | None, + request: ReceiveRequestT, + session: """BaseSession[ + SendRequestT, + SendNotificationT, + SendResultT, + ReceiveRequestT, + ReceiveNotificationT + ]""", + on_complete: Callable[["RequestResponder[ReceiveRequestT, SendResultT]"], Any], + ) -> None: + self.request_id = request_id + self.request_meta = request_meta + self.request = request + self._session = session + self._completed = False + self._on_complete = on_complete + self._entered = False # Track if we're in a context manager + + def __enter__(self) -> "RequestResponder[ReceiveRequestT, SendResultT]": + """Enter the context manager, enabling request cancellation tracking.""" + self._entered = True + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + """Exit the context manager, performing cleanup and notifying completion.""" + try: + if self._completed: + self._on_complete(self) + finally: + self._entered = False + + def respond(self, response: SendResultT | ErrorData) -> None: + """Send a response for this request. + + Must be called within a context manager block. + Raises: + RuntimeError: If not used within a context manager + AssertionError: If request was already responded to + """ + if not self._entered: + raise RuntimeError("RequestResponder must be used as a context manager") + assert not self._completed, "Request already responded to" + + self._completed = True + + self._session._send_response(request_id=self.request_id, response=response) + + def cancel(self) -> None: + """Cancel this request and mark it as completed.""" + if not self._entered: + raise RuntimeError("RequestResponder must be used as a context manager") + + self._completed = True # Mark as completed so it's removed from in_flight + # Send an error response to indicate cancellation + self._session._send_response( + request_id=self.request_id, + response=ErrorData(code=0, message="Request cancelled", data=None), + ) + + +class BaseSession( + Generic[ + SendRequestT, + SendNotificationT, + SendResultT, + ReceiveRequestT, + ReceiveNotificationT, + ], +): + """ + Implements an MCP "session" on top of read/write streams, including features + like request/response linking, notifications, and progress. + + This class is a context manager that automatically starts processing + messages when entered. + """ + + _response_streams: dict[RequestId, queue.Queue[JSONRPCResponse | JSONRPCError]] + _request_id: int + _in_flight: dict[RequestId, RequestResponder[ReceiveRequestT, SendResultT]] + _receive_request_type: type[ReceiveRequestT] + _receive_notification_type: type[ReceiveNotificationT] + + def __init__( + self, + read_stream: queue.Queue, + write_stream: queue.Queue, + receive_request_type: type[ReceiveRequestT], + receive_notification_type: type[ReceiveNotificationT], + # If none, reading will never time out + read_timeout_seconds: timedelta | None = None, + ) -> None: + self._read_stream = read_stream + self._write_stream = write_stream + self._response_streams = {} + self._request_id = 0 + self._receive_request_type = receive_request_type + self._receive_notification_type = receive_notification_type + self._session_read_timeout_seconds = read_timeout_seconds + self._in_flight = {} + self._exit_stack = ExitStack() + + def __enter__(self) -> Self: + self._executor = ThreadPoolExecutor() + self._receiver_future = self._executor.submit(self._receive_loop) + return self + + def check_receiver_status(self) -> None: + if self._receiver_future.done(): + self._receiver_future.result() + + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: + self._exit_stack.close() + self._read_stream.put(None) + self._write_stream.put(None) + + def send_request( + self, + request: SendRequestT, + result_type: type[ReceiveResultT], + request_read_timeout_seconds: timedelta | None = None, + metadata: MessageMetadata = None, + ) -> ReceiveResultT: + """ + Sends a request and wait for a response. Raises an McpError if the + response contains an error. If a request read timeout is provided, it + will take precedence over the session read timeout. + + Do not use this method to emit notifications! Use send_notification() + instead. + """ + self.check_receiver_status() + + request_id = self._request_id + self._request_id = request_id + 1 + + response_queue: queue.Queue[JSONRPCResponse | JSONRPCError] = queue.Queue() + self._response_streams[request_id] = response_queue + + try: + jsonrpc_request = JSONRPCRequest( + jsonrpc="2.0", + id=request_id, + **request.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + + self._write_stream.put(SessionMessage(message=JSONRPCMessage(jsonrpc_request), metadata=metadata)) + timeout = DEFAULT_RESPONSE_READ_TIMEOUT + if request_read_timeout_seconds is not None: + timeout = float(request_read_timeout_seconds.total_seconds()) + elif self._session_read_timeout_seconds is not None: + timeout = float(self._session_read_timeout_seconds.total_seconds()) + while True: + try: + response_or_error = response_queue.get(timeout=timeout) + break + except queue.Empty: + self.check_receiver_status() + continue + + if response_or_error is None: + raise MCPConnectionError( + ErrorData( + code=500, + message="No response received", + ) + ) + elif isinstance(response_or_error, JSONRPCError): + if response_or_error.error.code == 401: + raise MCPAuthError( + ErrorData(code=response_or_error.error.code, message=response_or_error.error.message) + ) + else: + raise MCPConnectionError( + ErrorData(code=response_or_error.error.code, message=response_or_error.error.message) + ) + else: + return result_type.model_validate(response_or_error.result) + + finally: + self._response_streams.pop(request_id, None) + + def send_notification( + self, + notification: SendNotificationT, + related_request_id: RequestId | None = None, + ) -> None: + """ + Emits a notification, which is a one-way message that does not expect + a response. + """ + self.check_receiver_status() + + # Some transport implementations may need to set the related_request_id + # to attribute to the notifications to the request that triggered them. + jsonrpc_notification = JSONRPCNotification( + jsonrpc="2.0", + **notification.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + session_message = SessionMessage( + message=JSONRPCMessage(jsonrpc_notification), + metadata=ServerMessageMetadata(related_request_id=related_request_id) if related_request_id else None, + ) + self._write_stream.put(session_message) + + def _send_response(self, request_id: RequestId, response: SendResultT | ErrorData) -> None: + if isinstance(response, ErrorData): + jsonrpc_error = JSONRPCError(jsonrpc="2.0", id=request_id, error=response) + session_message = SessionMessage(message=JSONRPCMessage(jsonrpc_error)) + self._write_stream.put(session_message) + else: + jsonrpc_response = JSONRPCResponse( + jsonrpc="2.0", + id=request_id, + result=response.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + session_message = SessionMessage(message=JSONRPCMessage(jsonrpc_response)) + self._write_stream.put(session_message) + + def _receive_loop(self) -> None: + """ + Main message processing loop. + In a real synchronous implementation, this would likely run in a separate thread. + """ + while True: + try: + # Attempt to receive a message (this would be blocking in a synchronous context) + message = self._read_stream.get(timeout=DEFAULT_RESPONSE_READ_TIMEOUT) + if message is None: + break + if isinstance(message, HTTPStatusError): + response_queue = self._response_streams.get(self._request_id - 1) + if response_queue is not None: + response_queue.put( + JSONRPCError( + jsonrpc="2.0", + id=self._request_id - 1, + error=ErrorData(code=message.response.status_code, message=message.args[0]), + ) + ) + else: + self._handle_incoming(RuntimeError(f"Received response with an unknown request ID: {message}")) + elif isinstance(message, Exception): + self._handle_incoming(message) + elif isinstance(message.message.root, JSONRPCRequest): + validated_request = self._receive_request_type.model_validate( + message.message.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + + responder = RequestResponder( + request_id=message.message.root.id, + request_meta=validated_request.root.params.meta if validated_request.root.params else None, + request=validated_request, + session=self, + on_complete=lambda r: self._in_flight.pop(r.request_id, None), + ) + + self._in_flight[responder.request_id] = responder + self._received_request(responder) + + if not responder._completed: + self._handle_incoming(responder) + + elif isinstance(message.message.root, JSONRPCNotification): + try: + notification = self._receive_notification_type.model_validate( + message.message.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + # Handle cancellation notifications + if isinstance(notification.root, CancelledNotification): + cancelled_id = notification.root.params.requestId + if cancelled_id in self._in_flight: + self._in_flight[cancelled_id].cancel() + else: + self._received_notification(notification) + self._handle_incoming(notification) + except Exception as e: + # For other validation errors, log and continue + logging.warning(f"Failed to validate notification: {e}. Message was: {message.message.root}") + else: # Response or error + response_queue = self._response_streams.get(message.message.root.id) + if response_queue is not None: + response_queue.put(message.message.root) + else: + self._handle_incoming(RuntimeError(f"Server Error: {message}")) + except queue.Empty: + continue + except Exception as e: + logging.exception("Error in message processing loop") + raise + + def _received_request(self, responder: RequestResponder[ReceiveRequestT, SendResultT]) -> None: + """ + Can be overridden by subclasses to handle a request without needing to + listen on the message stream. + + If the request is responded to within this method, it will not be + forwarded on to the message stream. + """ + pass + + def _received_notification(self, notification: ReceiveNotificationT) -> None: + """ + Can be overridden by subclasses to handle a notification without needing + to listen on the message stream. + """ + pass + + def send_progress_notification( + self, progress_token: str | int, progress: float, total: float | None = None + ) -> None: + """ + Sends a progress notification for a request that is currently being + processed. + """ + pass + + def _handle_incoming( + self, + req: RequestResponder[ReceiveRequestT, SendResultT] | ReceiveNotificationT | Exception, + ) -> None: + """A generic handler for incoming messages. Overwritten by subclasses.""" + pass diff --git a/api/core/mcp/session/client_session.py b/api/core/mcp/session/client_session.py new file mode 100644 index 0000000000..ed2ad508ab --- /dev/null +++ b/api/core/mcp/session/client_session.py @@ -0,0 +1,365 @@ +from datetime import timedelta +from typing import Any, Protocol + +from pydantic import AnyUrl, TypeAdapter + +from configs import dify_config +from core.mcp import types +from core.mcp.entities import SUPPORTED_PROTOCOL_VERSIONS, RequestContext +from core.mcp.session.base_session import BaseSession, RequestResponder + +DEFAULT_CLIENT_INFO = types.Implementation(name="Dify", version=dify_config.project.version) + + +class SamplingFnT(Protocol): + def __call__( + self, + context: RequestContext["ClientSession", Any], + params: types.CreateMessageRequestParams, + ) -> types.CreateMessageResult | types.ErrorData: ... + + +class ListRootsFnT(Protocol): + def __call__(self, context: RequestContext["ClientSession", Any]) -> types.ListRootsResult | types.ErrorData: ... + + +class LoggingFnT(Protocol): + def __call__( + self, + params: types.LoggingMessageNotificationParams, + ) -> None: ... + + +class MessageHandlerFnT(Protocol): + def __call__( + self, + message: RequestResponder[types.ServerRequest, types.ClientResult] | types.ServerNotification | Exception, + ) -> None: ... + + +def _default_message_handler( + message: RequestResponder[types.ServerRequest, types.ClientResult] | types.ServerNotification | Exception, +) -> None: + if isinstance(message, Exception): + raise ValueError(str(message)) + elif isinstance(message, (types.ServerNotification | RequestResponder)): + pass + + +def _default_sampling_callback( + context: RequestContext["ClientSession", Any], + params: types.CreateMessageRequestParams, +) -> types.CreateMessageResult | types.ErrorData: + return types.ErrorData( + code=types.INVALID_REQUEST, + message="Sampling not supported", + ) + + +def _default_list_roots_callback( + context: RequestContext["ClientSession", Any], +) -> types.ListRootsResult | types.ErrorData: + return types.ErrorData( + code=types.INVALID_REQUEST, + message="List roots not supported", + ) + + +def _default_logging_callback( + params: types.LoggingMessageNotificationParams, +) -> None: + pass + + +ClientResponse: TypeAdapter[types.ClientResult | types.ErrorData] = TypeAdapter(types.ClientResult | types.ErrorData) + + +class ClientSession( + BaseSession[ + types.ClientRequest, + types.ClientNotification, + types.ClientResult, + types.ServerRequest, + types.ServerNotification, + ] +): + def __init__( + self, + read_stream, + write_stream, + read_timeout_seconds: timedelta | None = None, + sampling_callback: SamplingFnT | None = None, + list_roots_callback: ListRootsFnT | None = None, + logging_callback: LoggingFnT | None = None, + message_handler: MessageHandlerFnT | None = None, + client_info: types.Implementation | None = None, + ) -> None: + super().__init__( + read_stream, + write_stream, + types.ServerRequest, + types.ServerNotification, + read_timeout_seconds=read_timeout_seconds, + ) + self._client_info = client_info or DEFAULT_CLIENT_INFO + self._sampling_callback = sampling_callback or _default_sampling_callback + self._list_roots_callback = list_roots_callback or _default_list_roots_callback + self._logging_callback = logging_callback or _default_logging_callback + self._message_handler = message_handler or _default_message_handler + + def initialize(self) -> types.InitializeResult: + sampling = types.SamplingCapability() + roots = types.RootsCapability( + # TODO: Should this be based on whether we + # _will_ send notifications, or only whether + # they're supported? + listChanged=True, + ) + + result = self.send_request( + types.ClientRequest( + types.InitializeRequest( + method="initialize", + params=types.InitializeRequestParams( + protocolVersion=types.LATEST_PROTOCOL_VERSION, + capabilities=types.ClientCapabilities( + sampling=sampling, + experimental=None, + roots=roots, + ), + clientInfo=self._client_info, + ), + ) + ), + types.InitializeResult, + ) + + if result.protocolVersion not in SUPPORTED_PROTOCOL_VERSIONS: + raise RuntimeError(f"Unsupported protocol version from the server: {result.protocolVersion}") + + self.send_notification( + types.ClientNotification(types.InitializedNotification(method="notifications/initialized")) + ) + + return result + + def send_ping(self) -> types.EmptyResult: + """Send a ping request.""" + return self.send_request( + types.ClientRequest( + types.PingRequest( + method="ping", + ) + ), + types.EmptyResult, + ) + + def send_progress_notification( + self, progress_token: str | int, progress: float, total: float | None = None + ) -> None: + """Send a progress notification.""" + self.send_notification( + types.ClientNotification( + types.ProgressNotification( + method="notifications/progress", + params=types.ProgressNotificationParams( + progressToken=progress_token, + progress=progress, + total=total, + ), + ), + ) + ) + + def set_logging_level(self, level: types.LoggingLevel) -> types.EmptyResult: + """Send a logging/setLevel request.""" + return self.send_request( + types.ClientRequest( + types.SetLevelRequest( + method="logging/setLevel", + params=types.SetLevelRequestParams(level=level), + ) + ), + types.EmptyResult, + ) + + def list_resources(self) -> types.ListResourcesResult: + """Send a resources/list request.""" + return self.send_request( + types.ClientRequest( + types.ListResourcesRequest( + method="resources/list", + ) + ), + types.ListResourcesResult, + ) + + def list_resource_templates(self) -> types.ListResourceTemplatesResult: + """Send a resources/templates/list request.""" + return self.send_request( + types.ClientRequest( + types.ListResourceTemplatesRequest( + method="resources/templates/list", + ) + ), + types.ListResourceTemplatesResult, + ) + + def read_resource(self, uri: AnyUrl) -> types.ReadResourceResult: + """Send a resources/read request.""" + return self.send_request( + types.ClientRequest( + types.ReadResourceRequest( + method="resources/read", + params=types.ReadResourceRequestParams(uri=uri), + ) + ), + types.ReadResourceResult, + ) + + def subscribe_resource(self, uri: AnyUrl) -> types.EmptyResult: + """Send a resources/subscribe request.""" + return self.send_request( + types.ClientRequest( + types.SubscribeRequest( + method="resources/subscribe", + params=types.SubscribeRequestParams(uri=uri), + ) + ), + types.EmptyResult, + ) + + def unsubscribe_resource(self, uri: AnyUrl) -> types.EmptyResult: + """Send a resources/unsubscribe request.""" + return self.send_request( + types.ClientRequest( + types.UnsubscribeRequest( + method="resources/unsubscribe", + params=types.UnsubscribeRequestParams(uri=uri), + ) + ), + types.EmptyResult, + ) + + def call_tool( + self, + name: str, + arguments: dict[str, Any] | None = None, + read_timeout_seconds: timedelta | None = None, + ) -> types.CallToolResult: + """Send a tools/call request.""" + + return self.send_request( + types.ClientRequest( + types.CallToolRequest( + method="tools/call", + params=types.CallToolRequestParams(name=name, arguments=arguments), + ) + ), + types.CallToolResult, + request_read_timeout_seconds=read_timeout_seconds, + ) + + def list_prompts(self) -> types.ListPromptsResult: + """Send a prompts/list request.""" + return self.send_request( + types.ClientRequest( + types.ListPromptsRequest( + method="prompts/list", + ) + ), + types.ListPromptsResult, + ) + + def get_prompt(self, name: str, arguments: dict[str, str] | None = None) -> types.GetPromptResult: + """Send a prompts/get request.""" + return self.send_request( + types.ClientRequest( + types.GetPromptRequest( + method="prompts/get", + params=types.GetPromptRequestParams(name=name, arguments=arguments), + ) + ), + types.GetPromptResult, + ) + + def complete( + self, + ref: types.ResourceReference | types.PromptReference, + argument: dict[str, str], + ) -> types.CompleteResult: + """Send a completion/complete request.""" + return self.send_request( + types.ClientRequest( + types.CompleteRequest( + method="completion/complete", + params=types.CompleteRequestParams( + ref=ref, + argument=types.CompletionArgument(**argument), + ), + ) + ), + types.CompleteResult, + ) + + def list_tools(self) -> types.ListToolsResult: + """Send a tools/list request.""" + return self.send_request( + types.ClientRequest( + types.ListToolsRequest( + method="tools/list", + ) + ), + types.ListToolsResult, + ) + + def send_roots_list_changed(self) -> None: + """Send a roots/list_changed notification.""" + self.send_notification( + types.ClientNotification( + types.RootsListChangedNotification( + method="notifications/roots/list_changed", + ) + ) + ) + + def _received_request(self, responder: RequestResponder[types.ServerRequest, types.ClientResult]) -> None: + ctx = RequestContext[ClientSession, Any]( + request_id=responder.request_id, + meta=responder.request_meta, + session=self, + lifespan_context=None, + ) + + match responder.request.root: + case types.CreateMessageRequest(params=params): + with responder: + response = self._sampling_callback(ctx, params) + client_response = ClientResponse.validate_python(response) + responder.respond(client_response) + + case types.ListRootsRequest(): + with responder: + list_roots_response = self._list_roots_callback(ctx) + client_response = ClientResponse.validate_python(list_roots_response) + responder.respond(client_response) + + case types.PingRequest(): + with responder: + return responder.respond(types.ClientResult(root=types.EmptyResult())) + + def _handle_incoming( + self, + req: RequestResponder[types.ServerRequest, types.ClientResult] | types.ServerNotification | Exception, + ) -> None: + """Handle incoming messages by forwarding to the message handler.""" + self._message_handler(req) + + def _received_notification(self, notification: types.ServerNotification) -> None: + """Handle notifications from the server.""" + # Process specific notification types + match notification.root: + case types.LoggingMessageNotification(params=params): + self._logging_callback(params) + case _: + pass diff --git a/api/core/mcp/types.py b/api/core/mcp/types.py new file mode 100644 index 0000000000..99d985a781 --- /dev/null +++ b/api/core/mcp/types.py @@ -0,0 +1,1217 @@ +from collections.abc import Callable +from dataclasses import dataclass +from typing import ( + Annotated, + Any, + Generic, + Literal, + Optional, + TypeAlias, + TypeVar, +) + +from pydantic import BaseModel, ConfigDict, Field, FileUrl, RootModel +from pydantic.networks import AnyUrl, UrlConstraints + +""" +Model Context Protocol bindings for Python + +These bindings were generated from https://github.com/modelcontextprotocol/specification, +using Claude, with a prompt something like the following: + +Generate idiomatic Python bindings for this schema for MCP, or the "Model Context +Protocol." The schema is defined in TypeScript, but there's also a JSON Schema version +for reference. + +* For the bindings, let's use Pydantic V2 models. +* Each model should allow extra fields everywhere, by specifying `model_config = + ConfigDict(extra='allow')`. Do this in every case, instead of a custom base class. +* Union types should be represented with a Pydantic `RootModel`. +* Define additional model classes instead of using dictionaries. Do this even if they're + not separate types in the schema. +""" +# Client support both version, not support 2025-06-18 yet. +LATEST_PROTOCOL_VERSION = "2025-03-26" +# Server support 2024-11-05 to allow claude to use. +SERVER_LATEST_PROTOCOL_VERSION = "2024-11-05" +ProgressToken = str | int +Cursor = str +Role = Literal["user", "assistant"] +RequestId = Annotated[int | str, Field(union_mode="left_to_right")] +AnyFunction: TypeAlias = Callable[..., Any] + + +class RequestParams(BaseModel): + class Meta(BaseModel): + progressToken: ProgressToken | None = None + """ + If specified, the caller requests out-of-band progress notifications for + this request (as represented by notifications/progress). The value of this + parameter is an opaque token that will be attached to any subsequent + notifications. The receiver is not obligated to provide these notifications. + """ + + model_config = ConfigDict(extra="allow") + + meta: Meta | None = Field(alias="_meta", default=None) + + +class NotificationParams(BaseModel): + class Meta(BaseModel): + model_config = ConfigDict(extra="allow") + + meta: Meta | None = Field(alias="_meta", default=None) + """ + This parameter name is reserved by MCP to allow clients and servers to attach + additional metadata to their notifications. + """ + + +RequestParamsT = TypeVar("RequestParamsT", bound=RequestParams | dict[str, Any] | None) +NotificationParamsT = TypeVar("NotificationParamsT", bound=NotificationParams | dict[str, Any] | None) +MethodT = TypeVar("MethodT", bound=str) + + +class Request(BaseModel, Generic[RequestParamsT, MethodT]): + """Base class for JSON-RPC requests.""" + + method: MethodT + params: RequestParamsT + model_config = ConfigDict(extra="allow") + + +class PaginatedRequest(Request[RequestParamsT, MethodT]): + cursor: Cursor | None = None + """ + An opaque token representing the current pagination position. + If provided, the server should return results starting after this cursor. + """ + + +class Notification(BaseModel, Generic[NotificationParamsT, MethodT]): + """Base class for JSON-RPC notifications.""" + + method: MethodT + params: NotificationParamsT + model_config = ConfigDict(extra="allow") + + +class Result(BaseModel): + """Base class for JSON-RPC results.""" + + model_config = ConfigDict(extra="allow") + + meta: dict[str, Any] | None = Field(alias="_meta", default=None) + """ + This result property is reserved by the protocol to allow clients and servers to + attach additional metadata to their responses. + """ + + +class PaginatedResult(Result): + nextCursor: Cursor | None = None + """ + An opaque token representing the pagination position after the last returned result. + If present, there may be more results available. + """ + + +class JSONRPCRequest(Request[dict[str, Any] | None, str]): + """A request that expects a response.""" + + jsonrpc: Literal["2.0"] + id: RequestId + method: str + params: dict[str, Any] | None = None + + +class JSONRPCNotification(Notification[dict[str, Any] | None, str]): + """A notification which does not expect a response.""" + + jsonrpc: Literal["2.0"] + params: dict[str, Any] | None = None + + +class JSONRPCResponse(BaseModel): + """A successful (non-error) response to a request.""" + + jsonrpc: Literal["2.0"] + id: RequestId + result: dict[str, Any] + model_config = ConfigDict(extra="allow") + + +# Standard JSON-RPC error codes +PARSE_ERROR = -32700 +INVALID_REQUEST = -32600 +METHOD_NOT_FOUND = -32601 +INVALID_PARAMS = -32602 +INTERNAL_ERROR = -32603 + + +class ErrorData(BaseModel): + """Error information for JSON-RPC error responses.""" + + code: int + """The error type that occurred.""" + + message: str + """ + A short description of the error. The message SHOULD be limited to a concise single + sentence. + """ + + data: Any | None = None + """ + Additional information about the error. The value of this member is defined by the + sender (e.g. detailed error information, nested errors etc.). + """ + + model_config = ConfigDict(extra="allow") + + +class JSONRPCError(BaseModel): + """A response to a request that indicates an error occurred.""" + + jsonrpc: Literal["2.0"] + id: str | int + error: ErrorData + model_config = ConfigDict(extra="allow") + + +class JSONRPCMessage(RootModel[JSONRPCRequest | JSONRPCNotification | JSONRPCResponse | JSONRPCError]): + pass + + +class EmptyResult(Result): + """A response that indicates success but carries no data.""" + + +class Implementation(BaseModel): + """Describes the name and version of an MCP implementation.""" + + name: str + version: str + model_config = ConfigDict(extra="allow") + + +class RootsCapability(BaseModel): + """Capability for root operations.""" + + listChanged: bool | None = None + """Whether the client supports notifications for changes to the roots list.""" + model_config = ConfigDict(extra="allow") + + +class SamplingCapability(BaseModel): + """Capability for logging operations.""" + + model_config = ConfigDict(extra="allow") + + +class ClientCapabilities(BaseModel): + """Capabilities a client may support.""" + + experimental: dict[str, dict[str, Any]] | None = None + """Experimental, non-standard capabilities that the client supports.""" + sampling: SamplingCapability | None = None + """Present if the client supports sampling from an LLM.""" + roots: RootsCapability | None = None + """Present if the client supports listing roots.""" + model_config = ConfigDict(extra="allow") + + +class PromptsCapability(BaseModel): + """Capability for prompts operations.""" + + listChanged: bool | None = None + """Whether this server supports notifications for changes to the prompt list.""" + model_config = ConfigDict(extra="allow") + + +class ResourcesCapability(BaseModel): + """Capability for resources operations.""" + + subscribe: bool | None = None + """Whether this server supports subscribing to resource updates.""" + listChanged: bool | None = None + """Whether this server supports notifications for changes to the resource list.""" + model_config = ConfigDict(extra="allow") + + +class ToolsCapability(BaseModel): + """Capability for tools operations.""" + + listChanged: bool | None = None + """Whether this server supports notifications for changes to the tool list.""" + model_config = ConfigDict(extra="allow") + + +class LoggingCapability(BaseModel): + """Capability for logging operations.""" + + model_config = ConfigDict(extra="allow") + + +class ServerCapabilities(BaseModel): + """Capabilities that a server may support.""" + + experimental: dict[str, dict[str, Any]] | None = None + """Experimental, non-standard capabilities that the server supports.""" + logging: LoggingCapability | None = None + """Present if the server supports sending log messages to the client.""" + prompts: PromptsCapability | None = None + """Present if the server offers any prompt templates.""" + resources: ResourcesCapability | None = None + """Present if the server offers any resources to read.""" + tools: ToolsCapability | None = None + """Present if the server offers any tools to call.""" + model_config = ConfigDict(extra="allow") + + +class InitializeRequestParams(RequestParams): + """Parameters for the initialize request.""" + + protocolVersion: str | int + """The latest version of the Model Context Protocol that the client supports.""" + capabilities: ClientCapabilities + clientInfo: Implementation + model_config = ConfigDict(extra="allow") + + +class InitializeRequest(Request[InitializeRequestParams, Literal["initialize"]]): + """ + This request is sent from the client to the server when it first connects, asking it + to begin initialization. + """ + + method: Literal["initialize"] + params: InitializeRequestParams + + +class InitializeResult(Result): + """After receiving an initialize request from the client, the server sends this.""" + + protocolVersion: str | int + """The version of the Model Context Protocol that the server wants to use.""" + capabilities: ServerCapabilities + serverInfo: Implementation + instructions: str | None = None + """Instructions describing how to use the server and its features.""" + + +class InitializedNotification(Notification[NotificationParams | None, Literal["notifications/initialized"]]): + """ + This notification is sent from the client to the server after initialization has + finished. + """ + + method: Literal["notifications/initialized"] + params: NotificationParams | None = None + + +class PingRequest(Request[RequestParams | None, Literal["ping"]]): + """ + A ping, issued by either the server or the client, to check that the other party is + still alive. + """ + + method: Literal["ping"] + params: RequestParams | None = None + + +class ProgressNotificationParams(NotificationParams): + """Parameters for progress notifications.""" + + progressToken: ProgressToken + """ + The progress token which was given in the initial request, used to associate this + notification with the request that is proceeding. + """ + progress: float + """ + The progress thus far. This should increase every time progress is made, even if the + total is unknown. + """ + total: float | None = None + """Total number of items to process (or total progress required), if known.""" + model_config = ConfigDict(extra="allow") + + +class ProgressNotification(Notification[ProgressNotificationParams, Literal["notifications/progress"]]): + """ + An out-of-band notification used to inform the receiver of a progress update for a + long-running request. + """ + + method: Literal["notifications/progress"] + params: ProgressNotificationParams + + +class ListResourcesRequest(PaginatedRequest[RequestParams | None, Literal["resources/list"]]): + """Sent from the client to request a list of resources the server has.""" + + method: Literal["resources/list"] + params: RequestParams | None = None + + +class Annotations(BaseModel): + audience: list[Role] | None = None + priority: Annotated[float, Field(ge=0.0, le=1.0)] | None = None + model_config = ConfigDict(extra="allow") + + +class Resource(BaseModel): + """A known resource that the server is capable of reading.""" + + uri: Annotated[AnyUrl, UrlConstraints(host_required=False)] + """The URI of this resource.""" + name: str + """A human-readable name for this resource.""" + description: str | None = None + """A description of what this resource represents.""" + mimeType: str | None = None + """The MIME type of this resource, if known.""" + size: int | None = None + """ + The size of the raw resource content, in bytes (i.e., before base64 encoding + or any tokenization), if known. + + This can be used by Hosts to display file sizes and estimate context window usage. + """ + annotations: Annotations | None = None + model_config = ConfigDict(extra="allow") + + +class ResourceTemplate(BaseModel): + """A template description for resources available on the server.""" + + uriTemplate: str + """ + A URI template (according to RFC 6570) that can be used to construct resource + URIs. + """ + name: str + """A human-readable name for the type of resource this template refers to.""" + description: str | None = None + """A human-readable description of what this template is for.""" + mimeType: str | None = None + """ + The MIME type for all resources that match this template. This should only be + included if all resources matching this template have the same type. + """ + annotations: Annotations | None = None + model_config = ConfigDict(extra="allow") + + +class ListResourcesResult(PaginatedResult): + """The server's response to a resources/list request from the client.""" + + resources: list[Resource] + + +class ListResourceTemplatesRequest(PaginatedRequest[RequestParams | None, Literal["resources/templates/list"]]): + """Sent from the client to request a list of resource templates the server has.""" + + method: Literal["resources/templates/list"] + params: RequestParams | None = None + + +class ListResourceTemplatesResult(PaginatedResult): + """The server's response to a resources/templates/list request from the client.""" + + resourceTemplates: list[ResourceTemplate] + + +class ReadResourceRequestParams(RequestParams): + """Parameters for reading a resource.""" + + uri: Annotated[AnyUrl, UrlConstraints(host_required=False)] + """ + The URI of the resource to read. The URI can use any protocol; it is up to the + server how to interpret it. + """ + model_config = ConfigDict(extra="allow") + + +class ReadResourceRequest(Request[ReadResourceRequestParams, Literal["resources/read"]]): + """Sent from the client to the server, to read a specific resource URI.""" + + method: Literal["resources/read"] + params: ReadResourceRequestParams + + +class ResourceContents(BaseModel): + """The contents of a specific resource or sub-resource.""" + + uri: Annotated[AnyUrl, UrlConstraints(host_required=False)] + """The URI of this resource.""" + mimeType: str | None = None + """The MIME type of this resource, if known.""" + model_config = ConfigDict(extra="allow") + + +class TextResourceContents(ResourceContents): + """Text contents of a resource.""" + + text: str + """ + The text of the item. This must only be set if the item can actually be represented + as text (not binary data). + """ + + +class BlobResourceContents(ResourceContents): + """Binary contents of a resource.""" + + blob: str + """A base64-encoded string representing the binary data of the item.""" + + +class ReadResourceResult(Result): + """The server's response to a resources/read request from the client.""" + + contents: list[TextResourceContents | BlobResourceContents] + + +class ResourceListChangedNotification( + Notification[NotificationParams | None, Literal["notifications/resources/list_changed"]] +): + """ + An optional notification from the server to the client, informing it that the list + of resources it can read from has changed. + """ + + method: Literal["notifications/resources/list_changed"] + params: NotificationParams | None = None + + +class SubscribeRequestParams(RequestParams): + """Parameters for subscribing to a resource.""" + + uri: Annotated[AnyUrl, UrlConstraints(host_required=False)] + """ + The URI of the resource to subscribe to. The URI can use any protocol; it is up to + the server how to interpret it. + """ + model_config = ConfigDict(extra="allow") + + +class SubscribeRequest(Request[SubscribeRequestParams, Literal["resources/subscribe"]]): + """ + Sent from the client to request resources/updated notifications from the server + whenever a particular resource changes. + """ + + method: Literal["resources/subscribe"] + params: SubscribeRequestParams + + +class UnsubscribeRequestParams(RequestParams): + """Parameters for unsubscribing from a resource.""" + + uri: Annotated[AnyUrl, UrlConstraints(host_required=False)] + """The URI of the resource to unsubscribe from.""" + model_config = ConfigDict(extra="allow") + + +class UnsubscribeRequest(Request[UnsubscribeRequestParams, Literal["resources/unsubscribe"]]): + """ + Sent from the client to request cancellation of resources/updated notifications from + the server. + """ + + method: Literal["resources/unsubscribe"] + params: UnsubscribeRequestParams + + +class ResourceUpdatedNotificationParams(NotificationParams): + """Parameters for resource update notifications.""" + + uri: Annotated[AnyUrl, UrlConstraints(host_required=False)] + """ + The URI of the resource that has been updated. This might be a sub-resource of the + one that the client actually subscribed to. + """ + model_config = ConfigDict(extra="allow") + + +class ResourceUpdatedNotification( + Notification[ResourceUpdatedNotificationParams, Literal["notifications/resources/updated"]] +): + """ + A notification from the server to the client, informing it that a resource has + changed and may need to be read again. + """ + + method: Literal["notifications/resources/updated"] + params: ResourceUpdatedNotificationParams + + +class ListPromptsRequest(PaginatedRequest[RequestParams | None, Literal["prompts/list"]]): + """Sent from the client to request a list of prompts and prompt templates.""" + + method: Literal["prompts/list"] + params: RequestParams | None = None + + +class PromptArgument(BaseModel): + """An argument for a prompt template.""" + + name: str + """The name of the argument.""" + description: str | None = None + """A human-readable description of the argument.""" + required: bool | None = None + """Whether this argument must be provided.""" + model_config = ConfigDict(extra="allow") + + +class Prompt(BaseModel): + """A prompt or prompt template that the server offers.""" + + name: str + """The name of the prompt or prompt template.""" + description: str | None = None + """An optional description of what this prompt provides.""" + arguments: list[PromptArgument] | None = None + """A list of arguments to use for templating the prompt.""" + model_config = ConfigDict(extra="allow") + + +class ListPromptsResult(PaginatedResult): + """The server's response to a prompts/list request from the client.""" + + prompts: list[Prompt] + + +class GetPromptRequestParams(RequestParams): + """Parameters for getting a prompt.""" + + name: str + """The name of the prompt or prompt template.""" + arguments: dict[str, str] | None = None + """Arguments to use for templating the prompt.""" + model_config = ConfigDict(extra="allow") + + +class GetPromptRequest(Request[GetPromptRequestParams, Literal["prompts/get"]]): + """Used by the client to get a prompt provided by the server.""" + + method: Literal["prompts/get"] + params: GetPromptRequestParams + + +class TextContent(BaseModel): + """Text content for a message.""" + + type: Literal["text"] + text: str + """The text content of the message.""" + annotations: Annotations | None = None + model_config = ConfigDict(extra="allow") + + +class ImageContent(BaseModel): + """Image content for a message.""" + + type: Literal["image"] + data: str + """The base64-encoded image data.""" + mimeType: str + """ + The MIME type of the image. Different providers may support different + image types. + """ + annotations: Annotations | None = None + model_config = ConfigDict(extra="allow") + + +class SamplingMessage(BaseModel): + """Describes a message issued to or received from an LLM API.""" + + role: Role + content: TextContent | ImageContent + model_config = ConfigDict(extra="allow") + + +class EmbeddedResource(BaseModel): + """ + The contents of a resource, embedded into a prompt or tool call result. + + It is up to the client how best to render embedded resources for the benefit + of the LLM and/or the user. + """ + + type: Literal["resource"] + resource: TextResourceContents | BlobResourceContents + annotations: Annotations | None = None + model_config = ConfigDict(extra="allow") + + +class PromptMessage(BaseModel): + """Describes a message returned as part of a prompt.""" + + role: Role + content: TextContent | ImageContent | EmbeddedResource + model_config = ConfigDict(extra="allow") + + +class GetPromptResult(Result): + """The server's response to a prompts/get request from the client.""" + + description: str | None = None + """An optional description for the prompt.""" + messages: list[PromptMessage] + + +class PromptListChangedNotification( + Notification[NotificationParams | None, Literal["notifications/prompts/list_changed"]] +): + """ + An optional notification from the server to the client, informing it that the list + of prompts it offers has changed. + """ + + method: Literal["notifications/prompts/list_changed"] + params: NotificationParams | None = None + + +class ListToolsRequest(PaginatedRequest[RequestParams | None, Literal["tools/list"]]): + """Sent from the client to request a list of tools the server has.""" + + method: Literal["tools/list"] + params: RequestParams | None = None + + +class ToolAnnotations(BaseModel): + """ + Additional properties describing a Tool to clients. + + NOTE: all properties in ToolAnnotations are **hints**. + They are not guaranteed to provide a faithful description of + tool behavior (including descriptive properties like `title`). + + Clients should never make tool use decisions based on ToolAnnotations + received from untrusted servers. + """ + + title: str | None = None + """A human-readable title for the tool.""" + + readOnlyHint: bool | None = None + """ + If true, the tool does not modify its environment. + Default: false + """ + + destructiveHint: bool | None = None + """ + If true, the tool may perform destructive updates to its environment. + If false, the tool performs only additive updates. + (This property is meaningful only when `readOnlyHint == false`) + Default: true + """ + + idempotentHint: bool | None = None + """ + If true, calling the tool repeatedly with the same arguments + will have no additional effect on the its environment. + (This property is meaningful only when `readOnlyHint == false`) + Default: false + """ + + openWorldHint: bool | None = None + """ + If true, this tool may interact with an "open world" of external + entities. If false, the tool's domain of interaction is closed. + For example, the world of a web search tool is open, whereas that + of a memory tool is not. + Default: true + """ + model_config = ConfigDict(extra="allow") + + +class Tool(BaseModel): + """Definition for a tool the client can call.""" + + name: str + """The name of the tool.""" + description: str | None = None + """A human-readable description of the tool.""" + inputSchema: dict[str, Any] + """A JSON Schema object defining the expected parameters for the tool.""" + annotations: ToolAnnotations | None = None + """Optional additional tool information.""" + model_config = ConfigDict(extra="allow") + + +class ListToolsResult(PaginatedResult): + """The server's response to a tools/list request from the client.""" + + tools: list[Tool] + + +class CallToolRequestParams(RequestParams): + """Parameters for calling a tool.""" + + name: str + arguments: dict[str, Any] | None = None + model_config = ConfigDict(extra="allow") + + +class CallToolRequest(Request[CallToolRequestParams, Literal["tools/call"]]): + """Used by the client to invoke a tool provided by the server.""" + + method: Literal["tools/call"] + params: CallToolRequestParams + + +class CallToolResult(Result): + """The server's response to a tool call.""" + + content: list[TextContent | ImageContent | EmbeddedResource] + isError: bool = False + + +class ToolListChangedNotification(Notification[NotificationParams | None, Literal["notifications/tools/list_changed"]]): + """ + An optional notification from the server to the client, informing it that the list + of tools it offers has changed. + """ + + method: Literal["notifications/tools/list_changed"] + params: NotificationParams | None = None + + +LoggingLevel = Literal["debug", "info", "notice", "warning", "error", "critical", "alert", "emergency"] + + +class SetLevelRequestParams(RequestParams): + """Parameters for setting the logging level.""" + + level: LoggingLevel + """The level of logging that the client wants to receive from the server.""" + model_config = ConfigDict(extra="allow") + + +class SetLevelRequest(Request[SetLevelRequestParams, Literal["logging/setLevel"]]): + """A request from the client to the server, to enable or adjust logging.""" + + method: Literal["logging/setLevel"] + params: SetLevelRequestParams + + +class LoggingMessageNotificationParams(NotificationParams): + """Parameters for logging message notifications.""" + + level: LoggingLevel + """The severity of this log message.""" + logger: str | None = None + """An optional name of the logger issuing this message.""" + data: Any + """ + The data to be logged, such as a string message or an object. Any JSON serializable + type is allowed here. + """ + model_config = ConfigDict(extra="allow") + + +class LoggingMessageNotification(Notification[LoggingMessageNotificationParams, Literal["notifications/message"]]): + """Notification of a log message passed from server to client.""" + + method: Literal["notifications/message"] + params: LoggingMessageNotificationParams + + +IncludeContext = Literal["none", "thisServer", "allServers"] + + +class ModelHint(BaseModel): + """Hints to use for model selection.""" + + name: str | None = None + """A hint for a model name.""" + + model_config = ConfigDict(extra="allow") + + +class ModelPreferences(BaseModel): + """ + The server's preferences for model selection, requested by the client during + sampling. + + Because LLMs can vary along multiple dimensions, choosing the "best" model is + rarely straightforward. Different models excel in different areas—some are + faster but less capable, others are more capable but more expensive, and so + on. This interface allows servers to express their priorities across multiple + dimensions to help clients make an appropriate selection for their use case. + + These preferences are always advisory. The client MAY ignore them. It is also + up to the client to decide how to interpret these preferences and how to + balance them against other considerations. + """ + + hints: list[ModelHint] | None = None + """ + Optional hints to use for model selection. + + If multiple hints are specified, the client MUST evaluate them in order + (such that the first match is taken). + + The client SHOULD prioritize these hints over the numeric priorities, but + MAY still use the priorities to select from ambiguous matches. + """ + + costPriority: float | None = None + """ + How much to prioritize cost when selecting a model. A value of 0 means cost + is not important, while a value of 1 means cost is the most important + factor. + """ + + speedPriority: float | None = None + """ + How much to prioritize sampling speed (latency) when selecting a model. A + value of 0 means speed is not important, while a value of 1 means speed is + the most important factor. + """ + + intelligencePriority: float | None = None + """ + How much to prioritize intelligence and capabilities when selecting a + model. A value of 0 means intelligence is not important, while a value of 1 + means intelligence is the most important factor. + """ + + model_config = ConfigDict(extra="allow") + + +class CreateMessageRequestParams(RequestParams): + """Parameters for creating a message.""" + + messages: list[SamplingMessage] + modelPreferences: ModelPreferences | None = None + """ + The server's preferences for which model to select. The client MAY ignore + these preferences. + """ + systemPrompt: str | None = None + """An optional system prompt the server wants to use for sampling.""" + includeContext: IncludeContext | None = None + """ + A request to include context from one or more MCP servers (including the caller), to + be attached to the prompt. + """ + temperature: float | None = None + maxTokens: int + """The maximum number of tokens to sample, as requested by the server.""" + stopSequences: list[str] | None = None + metadata: dict[str, Any] | None = None + """Optional metadata to pass through to the LLM provider.""" + model_config = ConfigDict(extra="allow") + + +class CreateMessageRequest(Request[CreateMessageRequestParams, Literal["sampling/createMessage"]]): + """A request from the server to sample an LLM via the client.""" + + method: Literal["sampling/createMessage"] + params: CreateMessageRequestParams + + +StopReason = Literal["endTurn", "stopSequence", "maxTokens"] | str + + +class CreateMessageResult(Result): + """The client's response to a sampling/create_message request from the server.""" + + role: Role + content: TextContent | ImageContent + model: str + """The name of the model that generated the message.""" + stopReason: StopReason | None = None + """The reason why sampling stopped, if known.""" + + +class ResourceReference(BaseModel): + """A reference to a resource or resource template definition.""" + + type: Literal["ref/resource"] + uri: str + """The URI or URI template of the resource.""" + model_config = ConfigDict(extra="allow") + + +class PromptReference(BaseModel): + """Identifies a prompt.""" + + type: Literal["ref/prompt"] + name: str + """The name of the prompt or prompt template""" + model_config = ConfigDict(extra="allow") + + +class CompletionArgument(BaseModel): + """The argument's information for completion requests.""" + + name: str + """The name of the argument""" + value: str + """The value of the argument to use for completion matching.""" + model_config = ConfigDict(extra="allow") + + +class CompleteRequestParams(RequestParams): + """Parameters for completion requests.""" + + ref: ResourceReference | PromptReference + argument: CompletionArgument + model_config = ConfigDict(extra="allow") + + +class CompleteRequest(Request[CompleteRequestParams, Literal["completion/complete"]]): + """A request from the client to the server, to ask for completion options.""" + + method: Literal["completion/complete"] + params: CompleteRequestParams + + +class Completion(BaseModel): + """Completion information.""" + + values: list[str] + """An array of completion values. Must not exceed 100 items.""" + total: int | None = None + """ + The total number of completion options available. This can exceed the number of + values actually sent in the response. + """ + hasMore: bool | None = None + """ + Indicates whether there are additional completion options beyond those provided in + the current response, even if the exact total is unknown. + """ + model_config = ConfigDict(extra="allow") + + +class CompleteResult(Result): + """The server's response to a completion/complete request""" + + completion: Completion + + +class ListRootsRequest(Request[RequestParams | None, Literal["roots/list"]]): + """ + Sent from the server to request a list of root URIs from the client. Roots allow + servers to ask for specific directories or files to operate on. A common example + for roots is providing a set of repositories or directories a server should operate + on. + + This request is typically used when the server needs to understand the file system + structure or access specific locations that the client has permission to read from. + """ + + method: Literal["roots/list"] + params: RequestParams | None = None + + +class Root(BaseModel): + """Represents a root directory or file that the server can operate on.""" + + uri: FileUrl + """ + The URI identifying the root. This *must* start with file:// for now. + This restriction may be relaxed in future versions of the protocol to allow + other URI schemes. + """ + name: str | None = None + """ + An optional name for the root. This can be used to provide a human-readable + identifier for the root, which may be useful for display purposes or for + referencing the root in other parts of the application. + """ + model_config = ConfigDict(extra="allow") + + +class ListRootsResult(Result): + """ + The client's response to a roots/list request from the server. + This result contains an array of Root objects, each representing a root directory + or file that the server can operate on. + """ + + roots: list[Root] + + +class RootsListChangedNotification( + Notification[NotificationParams | None, Literal["notifications/roots/list_changed"]] +): + """ + A notification from the client to the server, informing it that the list of + roots has changed. + + This notification should be sent whenever the client adds, removes, or + modifies any root. The server should then request an updated list of roots + using the ListRootsRequest. + """ + + method: Literal["notifications/roots/list_changed"] + params: NotificationParams | None = None + + +class CancelledNotificationParams(NotificationParams): + """Parameters for cancellation notifications.""" + + requestId: RequestId + """The ID of the request to cancel.""" + reason: str | None = None + """An optional string describing the reason for the cancellation.""" + model_config = ConfigDict(extra="allow") + + +class CancelledNotification(Notification[CancelledNotificationParams, Literal["notifications/cancelled"]]): + """ + This notification can be sent by either side to indicate that it is canceling a + previously-issued request. + """ + + method: Literal["notifications/cancelled"] + params: CancelledNotificationParams + + +class ClientRequest( + RootModel[ + PingRequest + | InitializeRequest + | CompleteRequest + | SetLevelRequest + | GetPromptRequest + | ListPromptsRequest + | ListResourcesRequest + | ListResourceTemplatesRequest + | ReadResourceRequest + | SubscribeRequest + | UnsubscribeRequest + | CallToolRequest + | ListToolsRequest + ] +): + pass + + +class ClientNotification( + RootModel[CancelledNotification | ProgressNotification | InitializedNotification | RootsListChangedNotification] +): + pass + + +class ClientResult(RootModel[EmptyResult | CreateMessageResult | ListRootsResult]): + pass + + +class ServerRequest(RootModel[PingRequest | CreateMessageRequest | ListRootsRequest]): + pass + + +class ServerNotification( + RootModel[ + CancelledNotification + | ProgressNotification + | LoggingMessageNotification + | ResourceUpdatedNotification + | ResourceListChangedNotification + | ToolListChangedNotification + | PromptListChangedNotification + ] +): + pass + + +class ServerResult( + RootModel[ + EmptyResult + | InitializeResult + | CompleteResult + | GetPromptResult + | ListPromptsResult + | ListResourcesResult + | ListResourceTemplatesResult + | ReadResourceResult + | CallToolResult + | ListToolsResult + ] +): + pass + + +ResumptionToken = str + +ResumptionTokenUpdateCallback = Callable[[ResumptionToken], None] + + +@dataclass +class ClientMessageMetadata: + """Metadata specific to client messages.""" + + resumption_token: ResumptionToken | None = None + on_resumption_token_update: Callable[[ResumptionToken], None] | None = None + + +@dataclass +class ServerMessageMetadata: + """Metadata specific to server messages.""" + + related_request_id: RequestId | None = None + request_context: object | None = None + + +MessageMetadata = ClientMessageMetadata | ServerMessageMetadata | None + + +@dataclass +class SessionMessage: + """A message with specific metadata for transport-specific features.""" + + message: JSONRPCMessage + metadata: MessageMetadata = None + + +class OAuthClientMetadata(BaseModel): + client_name: str + redirect_uris: list[str] + grant_types: Optional[list[str]] = None + response_types: Optional[list[str]] = None + token_endpoint_auth_method: Optional[str] = None + client_uri: Optional[str] = None + scope: Optional[str] = None + + +class OAuthClientInformation(BaseModel): + client_id: str + client_secret: Optional[str] = None + + +class OAuthClientInformationFull(OAuthClientInformation): + client_name: str | None = None + redirect_uris: list[str] + scope: Optional[str] = None + grant_types: Optional[list[str]] = None + response_types: Optional[list[str]] = None + token_endpoint_auth_method: Optional[str] = None + + +class OAuthTokens(BaseModel): + access_token: str + token_type: str + expires_in: Optional[int] = None + refresh_token: Optional[str] = None + scope: Optional[str] = None + + +class OAuthMetadata(BaseModel): + authorization_endpoint: str + token_endpoint: str + registration_endpoint: Optional[str] = None + response_types_supported: list[str] + grant_types_supported: Optional[list[str]] = None + code_challenge_methods_supported: Optional[list[str]] = None diff --git a/api/core/mcp/utils.py b/api/core/mcp/utils.py new file mode 100644 index 0000000000..a54badcd4c --- /dev/null +++ b/api/core/mcp/utils.py @@ -0,0 +1,114 @@ +import json + +import httpx + +from configs import dify_config +from core.mcp.types import ErrorData, JSONRPCError +from core.model_runtime.utils.encoders import jsonable_encoder + +HTTP_REQUEST_NODE_SSL_VERIFY = dify_config.HTTP_REQUEST_NODE_SSL_VERIFY + +STATUS_FORCELIST = [429, 500, 502, 503, 504] + + +def create_ssrf_proxy_mcp_http_client( + headers: dict[str, str] | None = None, + timeout: httpx.Timeout | None = None, +) -> httpx.Client: + """Create an HTTPX client with SSRF proxy configuration for MCP connections. + + Args: + headers: Optional headers to include in the client + timeout: Optional timeout configuration + + Returns: + Configured httpx.Client with proxy settings + """ + if dify_config.SSRF_PROXY_ALL_URL: + return httpx.Client( + verify=HTTP_REQUEST_NODE_SSL_VERIFY, + headers=headers or {}, + timeout=timeout, + follow_redirects=True, + proxy=dify_config.SSRF_PROXY_ALL_URL, + ) + elif dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL: + proxy_mounts = { + "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY), + "https://": httpx.HTTPTransport( + proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY + ), + } + return httpx.Client( + verify=HTTP_REQUEST_NODE_SSL_VERIFY, + headers=headers or {}, + timeout=timeout, + follow_redirects=True, + mounts=proxy_mounts, + ) + else: + return httpx.Client( + verify=HTTP_REQUEST_NODE_SSL_VERIFY, + headers=headers or {}, + timeout=timeout, + follow_redirects=True, + ) + + +def ssrf_proxy_sse_connect(url, **kwargs): + """Connect to SSE endpoint with SSRF proxy protection. + + This function creates an SSE connection using the configured proxy settings + to prevent SSRF attacks when connecting to external endpoints. + + Args: + url: The SSE endpoint URL + **kwargs: Additional arguments passed to the SSE connection + + Returns: + EventSource object for SSE streaming + """ + from httpx_sse import connect_sse + + # Extract client if provided, otherwise create one + client = kwargs.pop("client", None) + if client is None: + # Create client with SSRF proxy configuration + timeout = kwargs.pop( + "timeout", + httpx.Timeout( + timeout=dify_config.SSRF_DEFAULT_TIME_OUT, + connect=dify_config.SSRF_DEFAULT_CONNECT_TIME_OUT, + read=dify_config.SSRF_DEFAULT_READ_TIME_OUT, + write=dify_config.SSRF_DEFAULT_WRITE_TIME_OUT, + ), + ) + headers = kwargs.pop("headers", {}) + client = create_ssrf_proxy_mcp_http_client(headers=headers, timeout=timeout) + client_provided = False + else: + client_provided = True + + # Extract method if provided, default to GET + method = kwargs.pop("method", "GET") + + try: + return connect_sse(client, method, url, **kwargs) + except Exception: + # If we created the client, we need to clean it up on error + if not client_provided: + client.close() + raise + + +def create_mcp_error_response(request_id: int | str | None, code: int, message: str, data=None): + """Create MCP error response""" + error_data = ErrorData(code=code, message=message, data=data) + json_response = JSONRPCError( + jsonrpc="2.0", + id=request_id or 1, + error=error_data, + ) + json_data = json.dumps(jsonable_encoder(json_response)) + sse_content = f"event: message\ndata: {json_data}\n\n".encode() + yield sse_content diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 2254b3d4d5..a9f0a92e5d 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -1,6 +1,8 @@ from collections.abc import Sequence from typing import Optional +from sqlalchemy import select + from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.file import file_manager from core.model_manager import ModelInstance @@ -17,11 +19,15 @@ from core.prompt.utils.extract_thread_messages import extract_thread_messages from extensions.ext_database import db from factories import file_factory from models.model import AppMode, Conversation, Message, MessageFile -from models.workflow import WorkflowRun +from models.workflow import Workflow, WorkflowRun class TokenBufferMemory: - def __init__(self, conversation: Conversation, model_instance: ModelInstance) -> None: + def __init__( + self, + conversation: Conversation, + model_instance: ModelInstance, + ) -> None: self.conversation = conversation self.model_instance = model_instance @@ -36,20 +42,8 @@ class TokenBufferMemory: app_record = self.conversation.app # fetch limited messages, and return reversed - query = ( - db.session.query( - Message.id, - Message.query, - Message.answer, - Message.created_at, - Message.workflow_run_id, - Message.parent_message_id, - Message.answer_tokens, - ) - .filter( - Message.conversation_id == self.conversation.id, - ) - .order_by(Message.created_at.desc()) + stmt = ( + select(Message).where(Message.conversation_id == self.conversation.id).order_by(Message.created_at.desc()) ) if message_limit and message_limit > 0: @@ -57,7 +51,9 @@ class TokenBufferMemory: else: message_limit = 500 - messages = query.limit(message_limit).all() + stmt = stmt.limit(message_limit) + + messages = db.session.scalars(stmt).all() # instead of all messages from the conversation, we only need to extract messages # that belong to the thread of last message @@ -74,18 +70,20 @@ class TokenBufferMemory: files = db.session.query(MessageFile).filter(MessageFile.message_id == message.id).all() if files: file_extra_config = None - if self.conversation.mode not in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}: + if self.conversation.mode in {AppMode.AGENT_CHAT, AppMode.COMPLETION, AppMode.CHAT}: file_extra_config = FileUploadConfigManager.convert(self.conversation.model_config) + elif self.conversation.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}: + workflow_run = db.session.scalar( + select(WorkflowRun).where(WorkflowRun.id == message.workflow_run_id) + ) + if not workflow_run: + raise ValueError(f"Workflow run not found: {message.workflow_run_id}") + workflow = db.session.scalar(select(Workflow).where(Workflow.id == workflow_run.workflow_id)) + if not workflow: + raise ValueError(f"Workflow not found: {workflow_run.workflow_id}") + file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) else: - if message.workflow_run_id: - workflow_run = ( - db.session.query(WorkflowRun).filter(WorkflowRun.id == message.workflow_run_id).first() - ) - - if workflow_run and workflow_run.workflow: - file_extra_config = FileUploadConfigManager.convert( - workflow_run.workflow.features_dict, is_vision=False - ) + raise AssertionError(f"Invalid app mode: {self.conversation.mode}") detail = ImagePromptMessageContent.DETAIL.LOW if file_extra_config and app_record: diff --git a/api/core/model_runtime/entities/provider_entities.py b/api/core/model_runtime/entities/provider_entities.py index d0f9ee13e5..c9aa8d1474 100644 --- a/api/core/model_runtime/entities/provider_entities.py +++ b/api/core/model_runtime/entities/provider_entities.py @@ -123,6 +123,8 @@ class ProviderEntity(BaseModel): description: Optional[I18nObject] = None icon_small: Optional[I18nObject] = None icon_large: Optional[I18nObject] = None + icon_small_dark: Optional[I18nObject] = None + icon_large_dark: Optional[I18nObject] = None background: Optional[str] = None help: Optional[ProviderHelpEntity] = None supported_model_types: Sequence[ModelType] diff --git a/api/core/ops/langfuse_trace/langfuse_trace.py b/api/core/ops/langfuse_trace/langfuse_trace.py index a3dbce0e59..4a7e66d27c 100644 --- a/api/core/ops/langfuse_trace/langfuse_trace.py +++ b/api/core/ops/langfuse_trace/langfuse_trace.py @@ -28,7 +28,7 @@ from core.ops.langfuse_trace.entities.langfuse_trace_entity import ( UnitEnum, ) from core.ops.utils import filter_none_values -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.workflow.nodes.enums import NodeType from extensions.ext_database import db from models import EndUser, WorkflowNodeExecutionTriggeredFrom @@ -123,10 +123,10 @@ class LangFuseDataTrace(BaseTraceInstance): service_account = self.get_service_account_with_tenant(app_id) - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=service_account, - app_id=trace_info.metadata.get("app_id"), + app_id=app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) diff --git a/api/core/ops/langsmith_trace/langsmith_trace.py b/api/core/ops/langsmith_trace/langsmith_trace.py index f94e5e49d7..8a559c4929 100644 --- a/api/core/ops/langsmith_trace/langsmith_trace.py +++ b/api/core/ops/langsmith_trace/langsmith_trace.py @@ -27,7 +27,7 @@ from core.ops.langsmith_trace.entities.langsmith_trace_entity import ( LangSmithRunUpdateModel, ) from core.ops.utils import filter_none_values, generate_dotted_order -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey from core.workflow.nodes.enums import NodeType from extensions.ext_database import db @@ -145,10 +145,10 @@ class LangSmithDataTrace(BaseTraceInstance): service_account = self.get_service_account_with_tenant(app_id) - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=service_account, - app_id=trace_info.metadata.get("app_id"), + app_id=app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) diff --git a/api/core/ops/opik_trace/opik_trace.py b/api/core/ops/opik_trace/opik_trace.py index 8bedea20fb..be4997a5bf 100644 --- a/api/core/ops/opik_trace/opik_trace.py +++ b/api/core/ops/opik_trace/opik_trace.py @@ -21,7 +21,7 @@ from core.ops.entities.trace_entity import ( TraceTaskName, WorkflowTraceInfo, ) -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey from core.workflow.nodes.enums import NodeType from extensions.ext_database import db @@ -160,10 +160,10 @@ class OpikDataTrace(BaseTraceInstance): service_account = self.get_service_account_with_tenant(app_id) - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=service_account, - app_id=trace_info.metadata.get("app_id"), + app_id=app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) @@ -241,7 +241,7 @@ class OpikDataTrace(BaseTraceInstance): "trace_id": opik_trace_id, "id": prepare_opik_uuid(created_at, node_execution_id), "parent_span_id": prepare_opik_uuid(trace_info.start_time, parent_span_id), - "name": node_type, + "name": node_name, "type": run_type, "start_time": created_at, "end_time": finished_at, diff --git a/api/core/ops/weave_trace/weave_trace.py b/api/core/ops/weave_trace/weave_trace.py index 3917348a91..445c6a8741 100644 --- a/api/core/ops/weave_trace/weave_trace.py +++ b/api/core/ops/weave_trace/weave_trace.py @@ -22,7 +22,7 @@ from core.ops.entities.trace_entity import ( WorkflowTraceInfo, ) from core.ops.weave_trace.entities.weave_trace_entity import WeaveTraceModel -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey from core.workflow.nodes.enums import NodeType from extensions.ext_database import db @@ -144,10 +144,10 @@ class WeaveDataTrace(BaseTraceInstance): service_account = self.get_service_account_with_tenant(app_id) - workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository( + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=session_factory, user=service_account, - app_id=trace_info.metadata.get("app_id"), + app_id=app_id, triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) diff --git a/api/core/plugin/entities/parameters.py b/api/core/plugin/entities/parameters.py index 2b438a3c33..2be65d67a0 100644 --- a/api/core/plugin/entities/parameters.py +++ b/api/core/plugin/entities/parameters.py @@ -43,6 +43,19 @@ class PluginParameterType(enum.StrEnum): # deprecated, should not use. SYSTEM_FILES = CommonParameterType.SYSTEM_FILES.value + # MCP object and array type parameters + ARRAY = CommonParameterType.ARRAY.value + OBJECT = CommonParameterType.OBJECT.value + + +class MCPServerParameterType(enum.StrEnum): + """ + MCP server got complex parameter types + """ + + ARRAY = "array" + OBJECT = "object" + class PluginParameterAutoGenerate(BaseModel): class Type(enum.StrEnum): @@ -138,6 +151,34 @@ def cast_parameter_value(typ: enum.StrEnum, value: Any, /): if value and not isinstance(value, list): raise ValueError("The tools selector must be a list.") return value + case PluginParameterType.ARRAY: + if not isinstance(value, list): + # Try to parse JSON string for arrays + if isinstance(value, str): + try: + import json + + parsed_value = json.loads(value) + if isinstance(parsed_value, list): + return parsed_value + except (json.JSONDecodeError, ValueError): + pass + return [value] + return value + case PluginParameterType.OBJECT: + if not isinstance(value, dict): + # Try to parse JSON string for objects + if isinstance(value, str): + try: + import json + + parsed_value = json.loads(value) + if isinstance(parsed_value, dict): + return parsed_value + except (json.JSONDecodeError, ValueError): + pass + return {} + return value case _: return str(value) except ValueError: diff --git a/api/core/plugin/entities/plugin.py b/api/core/plugin/entities/plugin.py index bdf7d5ce1f..e5cf7ee03a 100644 --- a/api/core/plugin/entities/plugin.py +++ b/api/core/plugin/entities/plugin.py @@ -72,12 +72,14 @@ class PluginDeclaration(BaseModel): class Meta(BaseModel): minimum_dify_version: Optional[str] = Field(default=None, pattern=r"^\d{1,4}(\.\d{1,4}){1,3}(-\w{1,16})?$") + version: Optional[str] = Field(default=None) version: str = Field(..., pattern=r"^\d{1,4}(\.\d{1,4}){1,3}(-\w{1,16})?$") author: Optional[str] = Field(..., pattern=r"^[a-zA-Z0-9_-]{1,64}$") name: str = Field(..., pattern=r"^[a-z0-9_-]{1,128}$") description: I18nObject icon: str + icon_dark: Optional[str] = Field(default=None) label: I18nObject category: PluginCategory created_at: datetime.datetime diff --git a/api/core/plugin/entities/plugin_daemon.py b/api/core/plugin/entities/plugin_daemon.py index 592b42c0da..00253b8a11 100644 --- a/api/core/plugin/entities/plugin_daemon.py +++ b/api/core/plugin/entities/plugin_daemon.py @@ -53,6 +53,7 @@ class PluginAgentProviderEntity(BaseModel): plugin_unique_identifier: str plugin_id: str declaration: AgentProviderEntityWithPlugin + meta: PluginDeclaration.Meta class PluginBasicBooleanResponse(BaseModel): diff --git a/api/core/plugin/entities/request.py b/api/core/plugin/entities/request.py index f9c81ed4d5..89f595ec46 100644 --- a/api/core/plugin/entities/request.py +++ b/api/core/plugin/entities/request.py @@ -32,7 +32,7 @@ class RequestInvokeTool(BaseModel): Request to invoke a tool """ - tool_type: Literal["builtin", "workflow", "api"] + tool_type: Literal["builtin", "workflow", "api", "mcp"] provider: str tool: str tool_parameters: dict diff --git a/api/core/plugin/impl/plugin.py b/api/core/plugin/impl/plugin.py index b7f7b31655..04ac8c9649 100644 --- a/api/core/plugin/impl/plugin.py +++ b/api/core/plugin/impl/plugin.py @@ -36,7 +36,7 @@ class PluginInstaller(BasePluginClient): "GET", f"plugin/{tenant_id}/management/list", PluginListResponse, - params={"page": 1, "page_size": 256}, + params={"page": 1, "page_size": 256, "response_type": "paged"}, ) return result.list @@ -45,7 +45,7 @@ class PluginInstaller(BasePluginClient): "GET", f"plugin/{tenant_id}/management/list", PluginListResponse, - params={"page": page, "page_size": page_size}, + params={"page": page, "page_size": page_size, "response_type": "paged"}, ) def upload_pkg( diff --git a/api/core/prompt/utils/extract_thread_messages.py b/api/core/prompt/utils/extract_thread_messages.py index f7aef76c87..4b883622a7 100644 --- a/api/core/prompt/utils/extract_thread_messages.py +++ b/api/core/prompt/utils/extract_thread_messages.py @@ -1,10 +1,11 @@ -from typing import Any +from collections.abc import Sequence from constants import UUID_NIL +from models import Message -def extract_thread_messages(messages: list[Any]): - thread_messages = [] +def extract_thread_messages(messages: Sequence[Message]): + thread_messages: list[Message] = [] next_message = None for message in messages: diff --git a/api/core/prompt/utils/get_thread_messages_length.py b/api/core/prompt/utils/get_thread_messages_length.py index f49466db6d..de64c27a73 100644 --- a/api/core/prompt/utils/get_thread_messages_length.py +++ b/api/core/prompt/utils/get_thread_messages_length.py @@ -1,3 +1,5 @@ +from sqlalchemy import select + from core.prompt.utils.extract_thread_messages import extract_thread_messages from extensions.ext_database import db from models.model import Message @@ -8,19 +10,9 @@ def get_thread_messages_length(conversation_id: str) -> int: Get the number of thread messages based on the parent message id. """ # Fetch all messages related to the conversation - query = ( - db.session.query( - Message.id, - Message.parent_message_id, - Message.answer, - ) - .filter( - Message.conversation_id == conversation_id, - ) - .order_by(Message.created_at.desc()) - ) + stmt = select(Message).where(Message.conversation_id == conversation_id).order_by(Message.created_at.desc()) - messages = query.all() + messages = db.session.scalars(stmt).all() # Extract thread messages thread_messages = extract_thread_messages(messages) diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 2c5178241c..5a6903d3d5 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -3,7 +3,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import Optional from flask import Flask, current_app -from sqlalchemy.orm import load_only +from sqlalchemy.orm import Session, load_only from configs import dify_config from core.rag.data_post_processor.data_post_processor import DataPostProcessor @@ -144,7 +144,8 @@ class RetrievalService: @classmethod def _get_dataset(cls, dataset_id: str) -> Optional[Dataset]: - return db.session.query(Dataset).filter(Dataset.id == dataset_id).first() + with Session(db.engine) as session: + return session.query(Dataset).filter(Dataset.id == dataset_id).first() @classmethod def keyword_search( diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index 8ce194c683..05fa73011a 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -47,6 +47,7 @@ class QdrantConfig(BaseModel): grpc_port: int = 6334 prefer_grpc: bool = False replication_factor: int = 1 + write_consistency_factor: int = 1 def to_qdrant_params(self): if self.endpoint and self.endpoint.startswith("path:"): @@ -127,6 +128,7 @@ class QdrantVector(BaseVector): hnsw_config=hnsw_config, timeout=int(self._client_config.timeout), replication_factor=self._client_config.replication_factor, + write_consistency_factor=self._client_config.write_consistency_factor, ) # create group_id payload index diff --git a/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py b/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py index a124faa503..552068c99e 100644 --- a/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py +++ b/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py @@ -4,6 +4,7 @@ from typing import Any, Optional import tablestore # type: ignore from pydantic import BaseModel, model_validator +from tablestore import BatchGetRowRequest, TableInBatchGetRowItem from configs import dify_config from core.rag.datasource.vdb.field import Field @@ -50,6 +51,29 @@ class TableStoreVector(BaseVector): self._index_name = f"{collection_name}_idx" self._tags_field = f"{Field.METADATA_KEY.value}_tags" + def create_collection(self, embeddings: list[list[float]], **kwargs): + dimension = len(embeddings[0]) + self._create_collection(dimension) + + def get_by_ids(self, ids: list[str]) -> list[Document]: + docs = [] + request = BatchGetRowRequest() + columns_to_get = [Field.METADATA_KEY.value, Field.CONTENT_KEY.value] + rows_to_get = [[("id", _id)] for _id in ids] + request.add(TableInBatchGetRowItem(self._table_name, rows_to_get, columns_to_get, None, 1)) + + result = self._tablestore_client.batch_get_row(request) + table_result = result.get_result_by_table(self._table_name) + for item in table_result: + if item.is_ok and item.row: + kv = {k: v for k, v, t in item.row.attribute_columns} + docs.append( + Document( + page_content=kv[Field.CONTENT_KEY.value], metadata=json.loads(kv[Field.METADATA_KEY.value]) + ) + ) + return docs + def get_type(self) -> str: return VectorType.TABLESTORE diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index d2bf3eb92a..75afe0cdb8 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -122,7 +122,6 @@ class TencentVector(BaseVector): metric_type, params, ) - index_text = vdb_index.FilterIndex(self.field_text, enum.FieldType.String, enum.IndexType.FILTER) index_metadate = vdb_index.FilterIndex(self.field_metadata, enum.FieldType.Json, enum.IndexType.FILTER) index_sparse_vector = vdb_index.SparseIndex( name="sparse_vector", @@ -130,7 +129,7 @@ class TencentVector(BaseVector): index_type=enum.IndexType.SPARSE_INVERTED, metric_type=enum.MetricType.IP, ) - indexes = [index_id, index_vector, index_text, index_metadate] + indexes = [index_id, index_vector, index_metadate] if self._enable_hybrid_search: indexes.append(index_sparse_vector) try: @@ -149,7 +148,7 @@ class TencentVector(BaseVector): index_metadate = vdb_index.FilterIndex( self.field_metadata, enum.FieldType.String, enum.IndexType.FILTER ) - indexes = [index_id, index_vector, index_text, index_metadate] + indexes = [index_id, index_vector, index_metadate] if self._enable_hybrid_search: indexes.append(index_sparse_vector) self._client.create_collection( diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 67a4a515b1..00080b0fae 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -1,3 +1,5 @@ +import logging +import time from abc import ABC, abstractmethod from typing import Any, Optional @@ -13,6 +15,8 @@ from extensions.ext_database import db from extensions.ext_redis import redis_client from models.dataset import Dataset, Whitelist +logger = logging.getLogger(__name__) + class AbstractVectorFactory(ABC): @abstractmethod @@ -173,8 +177,20 @@ class Vector: def create(self, texts: Optional[list] = None, **kwargs): if texts: - embeddings = self._embeddings.embed_documents([document.page_content for document in texts]) - self._vector_processor.create(texts=texts, embeddings=embeddings, **kwargs) + start = time.time() + logger.info(f"start embedding {len(texts)} texts {start}") + batch_size = 1000 + total_batches = len(texts) + batch_size - 1 + for i in range(0, len(texts), batch_size): + batch = texts[i : i + batch_size] + batch_start = time.time() + logger.info(f"Processing batch {i // batch_size + 1}/{total_batches} ({len(batch)} texts)") + batch_embeddings = self._embeddings.embed_documents([document.page_content for document in batch]) + logger.info( + f"Embedding batch {i // batch_size + 1}/{total_batches} took {time.time() - batch_start:.3f}s" + ) + self._vector_processor.create(texts=batch, embeddings=batch_embeddings, **kwargs) + logger.info(f"Embedding {len(texts)} texts took {time.time() - start:.3f}s") def add_texts(self, documents: list[Document], **kwargs): if kwargs.get("duplicate_check", False): diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 3fca48be22..5c0360b064 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -9,6 +9,7 @@ from typing import Any, Optional, Union, cast from flask import Flask, current_app from sqlalchemy import Float, and_, or_, text from sqlalchemy import cast as sqlalchemy_cast +from sqlalchemy.orm import Session from core.app.app_config.entities import ( DatasetEntity, @@ -598,7 +599,8 @@ class DatasetRetrieval: metadata_condition: Optional[MetadataCondition] = None, ): with flask_app.app_context(): - dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first() + with Session(db.engine) as session: + dataset = session.query(Dataset).filter(Dataset.id == dataset_id).first() if not dataset: return [] diff --git a/api/core/repositories/__init__.py b/api/core/repositories/__init__.py index 6452317120..052ba1c2cb 100644 --- a/api/core/repositories/__init__.py +++ b/api/core/repositories/__init__.py @@ -5,8 +5,11 @@ This package contains concrete implementations of the repository interfaces defined in the core.workflow.repository package. """ +from core.repositories.factory import DifyCoreRepositoryFactory, RepositoryImportError from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository __all__ = [ + "DifyCoreRepositoryFactory", + "RepositoryImportError", "SQLAlchemyWorkflowNodeExecutionRepository", ] diff --git a/api/core/repositories/factory.py b/api/core/repositories/factory.py new file mode 100644 index 0000000000..4118aa61c7 --- /dev/null +++ b/api/core/repositories/factory.py @@ -0,0 +1,224 @@ +""" +Repository factory for dynamically creating repository instances based on configuration. + +This module provides a Django-like settings system for repository implementations, +allowing users to configure different repository backends through string paths. +""" + +import importlib +import inspect +import logging +from typing import Protocol, Union + +from sqlalchemy.engine import Engine +from sqlalchemy.orm import sessionmaker + +from configs import dify_config +from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository +from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository +from models import Account, EndUser +from models.enums import WorkflowRunTriggeredFrom +from models.workflow import WorkflowNodeExecutionTriggeredFrom + +logger = logging.getLogger(__name__) + + +class RepositoryImportError(Exception): + """Raised when a repository implementation cannot be imported or instantiated.""" + + pass + + +class DifyCoreRepositoryFactory: + """ + Factory for creating repository instances based on configuration. + + This factory supports Django-like settings where repository implementations + are specified as module paths (e.g., 'module.submodule.ClassName'). + """ + + @staticmethod + def _import_class(class_path: str) -> type: + """ + Import a class from a module path string. + + Args: + class_path: Full module path to the class (e.g., 'module.submodule.ClassName') + + Returns: + The imported class + + Raises: + RepositoryImportError: If the class cannot be imported + """ + try: + module_path, class_name = class_path.rsplit(".", 1) + module = importlib.import_module(module_path) + repo_class = getattr(module, class_name) + assert isinstance(repo_class, type) + return repo_class + except (ValueError, ImportError, AttributeError) as e: + raise RepositoryImportError(f"Cannot import repository class '{class_path}': {e}") from e + + @staticmethod + def _validate_repository_interface(repository_class: type, expected_interface: type[Protocol]) -> None: # type: ignore + """ + Validate that a class implements the expected repository interface. + + Args: + repository_class: The class to validate + expected_interface: The expected interface/protocol + + Raises: + RepositoryImportError: If the class doesn't implement the interface + """ + # Check if the class has all required methods from the protocol + required_methods = [ + method + for method in dir(expected_interface) + if not method.startswith("_") and callable(getattr(expected_interface, method, None)) + ] + + missing_methods = [] + for method_name in required_methods: + if not hasattr(repository_class, method_name): + missing_methods.append(method_name) + + if missing_methods: + raise RepositoryImportError( + f"Repository class '{repository_class.__name__}' does not implement required methods " + f"{missing_methods} from interface '{expected_interface.__name__}'" + ) + + @staticmethod + def _validate_constructor_signature(repository_class: type, required_params: list[str]) -> None: + """ + Validate that a repository class constructor accepts required parameters. + + Args: + repository_class: The class to validate + required_params: List of required parameter names + + Raises: + RepositoryImportError: If the constructor doesn't accept required parameters + """ + + try: + # MyPy may flag the line below with the following error: + # + # > Accessing "__init__" on an instance is unsound, since + # > instance.__init__ could be from an incompatible subclass. + # + # Despite this, we need to ensure that the constructor of `repository_class` + # has a compatible signature. + signature = inspect.signature(repository_class.__init__) # type: ignore[misc] + param_names = list(signature.parameters.keys()) + + # Remove 'self' parameter + if "self" in param_names: + param_names.remove("self") + + missing_params = [param for param in required_params if param not in param_names] + if missing_params: + raise RepositoryImportError( + f"Repository class '{repository_class.__name__}' constructor does not accept required parameters: " + f"{missing_params}. Expected parameters: {required_params}" + ) + except Exception as e: + raise RepositoryImportError( + f"Failed to validate constructor signature for '{repository_class.__name__}': {e}" + ) from e + + @classmethod + def create_workflow_execution_repository( + cls, + session_factory: Union[sessionmaker, Engine], + user: Union[Account, EndUser], + app_id: str, + triggered_from: WorkflowRunTriggeredFrom, + ) -> WorkflowExecutionRepository: + """ + Create a WorkflowExecutionRepository instance based on configuration. + + Args: + session_factory: SQLAlchemy sessionmaker or engine + user: Account or EndUser object + app_id: Application ID + triggered_from: Source of the execution trigger + + Returns: + Configured WorkflowExecutionRepository instance + + Raises: + RepositoryImportError: If the configured repository cannot be created + """ + class_path = dify_config.CORE_WORKFLOW_EXECUTION_REPOSITORY + logger.debug(f"Creating WorkflowExecutionRepository from: {class_path}") + + try: + repository_class = cls._import_class(class_path) + cls._validate_repository_interface(repository_class, WorkflowExecutionRepository) + cls._validate_constructor_signature( + repository_class, ["session_factory", "user", "app_id", "triggered_from"] + ) + + return repository_class( # type: ignore[no-any-return] + session_factory=session_factory, + user=user, + app_id=app_id, + triggered_from=triggered_from, + ) + except RepositoryImportError: + # Re-raise our custom errors as-is + raise + except Exception as e: + logger.exception("Failed to create WorkflowExecutionRepository") + raise RepositoryImportError(f"Failed to create WorkflowExecutionRepository from '{class_path}': {e}") from e + + @classmethod + def create_workflow_node_execution_repository( + cls, + session_factory: Union[sessionmaker, Engine], + user: Union[Account, EndUser], + app_id: str, + triggered_from: WorkflowNodeExecutionTriggeredFrom, + ) -> WorkflowNodeExecutionRepository: + """ + Create a WorkflowNodeExecutionRepository instance based on configuration. + + Args: + session_factory: SQLAlchemy sessionmaker or engine + user: Account or EndUser object + app_id: Application ID + triggered_from: Source of the execution trigger + + Returns: + Configured WorkflowNodeExecutionRepository instance + + Raises: + RepositoryImportError: If the configured repository cannot be created + """ + class_path = dify_config.CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY + logger.debug(f"Creating WorkflowNodeExecutionRepository from: {class_path}") + + try: + repository_class = cls._import_class(class_path) + cls._validate_repository_interface(repository_class, WorkflowNodeExecutionRepository) + cls._validate_constructor_signature( + repository_class, ["session_factory", "user", "app_id", "triggered_from"] + ) + + return repository_class( # type: ignore[no-any-return] + session_factory=session_factory, + user=user, + app_id=app_id, + triggered_from=triggered_from, + ) + except RepositoryImportError: + # Re-raise our custom errors as-is + raise + except Exception as e: + logger.exception("Failed to create WorkflowNodeExecutionRepository") + raise RepositoryImportError( + f"Failed to create WorkflowNodeExecutionRepository from '{class_path}': {e}" + ) from e diff --git a/api/core/repositories/sqlalchemy_workflow_execution_repository.py b/api/core/repositories/sqlalchemy_workflow_execution_repository.py index cdec92aee7..0b3e5eb424 100644 --- a/api/core/repositories/sqlalchemy_workflow_execution_repository.py +++ b/api/core/repositories/sqlalchemy_workflow_execution_repository.py @@ -17,6 +17,7 @@ from core.workflow.entities.workflow_execution import ( ) from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository from core.workflow.workflow_type_encoder import WorkflowRuntimeTypeConverter +from libs.helper import extract_tenant_id from models import ( Account, CreatorUserRole, @@ -67,7 +68,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository): ) # Extract tenant_id from user - tenant_id: str | None = user.tenant_id if isinstance(user, EndUser) else user.current_tenant_id + tenant_id = extract_tenant_id(user) if not tenant_id: raise ValueError("User must have a tenant_id or current_tenant_id") self._tenant_id = tenant_id diff --git a/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py b/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py index 797cce9354..a5feeb0d7c 100644 --- a/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py +++ b/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py @@ -20,6 +20,7 @@ from core.workflow.entities.workflow_node_execution import ( from core.workflow.nodes.enums import NodeType from core.workflow.repositories.workflow_node_execution_repository import OrderConfig, WorkflowNodeExecutionRepository from core.workflow.workflow_type_encoder import WorkflowRuntimeTypeConverter +from libs.helper import extract_tenant_id from models import ( Account, CreatorUserRole, @@ -70,7 +71,7 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository) ) # Extract tenant_id from user - tenant_id: str | None = user.tenant_id if isinstance(user, EndUser) else user.current_tenant_id + tenant_id = extract_tenant_id(user) if not tenant_id: raise ValueError("User must have a tenant_id or current_tenant_id") self._tenant_id = tenant_id diff --git a/api/core/tools/custom_tool/provider.py b/api/core/tools/custom_tool/provider.py index 3137d32013..fbe1d79137 100644 --- a/api/core/tools/custom_tool/provider.py +++ b/api/core/tools/custom_tool/provider.py @@ -39,19 +39,22 @@ class ApiToolProviderController(ToolProviderController): type=ProviderConfig.Type.SELECT, options=[ ProviderConfig.Option(value="none", label=I18nObject(en_US="None", zh_Hans="无")), - ProviderConfig.Option(value="api_key", label=I18nObject(en_US="api_key", zh_Hans="api_key")), + ProviderConfig.Option(value="api_key_header", label=I18nObject(en_US="Header", zh_Hans="请求头")), + ProviderConfig.Option( + value="api_key_query", label=I18nObject(en_US="Query Param", zh_Hans="查询参数") + ), ], default="none", help=I18nObject(en_US="The auth type of the api provider", zh_Hans="api provider 的认证类型"), ) ] - if auth_type == ApiProviderAuthType.API_KEY: + if auth_type == ApiProviderAuthType.API_KEY_HEADER: credentials_schema = [ *credentials_schema, ProviderConfig( name="api_key_header", required=False, - default="api_key", + default="Authorization", type=ProviderConfig.Type.TEXT_INPUT, help=I18nObject(en_US="The header name of the api key", zh_Hans="携带 api key 的 header 名称"), ), @@ -74,6 +77,25 @@ class ApiToolProviderController(ToolProviderController): ], ), ] + elif auth_type == ApiProviderAuthType.API_KEY_QUERY: + credentials_schema = [ + *credentials_schema, + ProviderConfig( + name="api_key_query_param", + required=False, + default="key", + type=ProviderConfig.Type.TEXT_INPUT, + help=I18nObject( + en_US="The query parameter name of the api key", zh_Hans="携带 api key 的查询参数名称" + ), + ), + ProviderConfig( + name="api_key_value", + required=True, + type=ProviderConfig.Type.SECRET_INPUT, + help=I18nObject(en_US="The api key", zh_Hans="api key 的值"), + ), + ] elif auth_type == ApiProviderAuthType.NONE: pass diff --git a/api/core/tools/custom_tool/tool.py b/api/core/tools/custom_tool/tool.py index 2f5cc6d4c0..10653b9948 100644 --- a/api/core/tools/custom_tool/tool.py +++ b/api/core/tools/custom_tool/tool.py @@ -78,8 +78,8 @@ class ApiTool(Tool): if "auth_type" not in credentials: raise ToolProviderCredentialValidationError("Missing auth_type") - if credentials["auth_type"] == "api_key": - api_key_header = "api_key" + if credentials["auth_type"] in ("api_key_header", "api_key"): # backward compatibility: + api_key_header = "Authorization" if "api_key_header" in credentials: api_key_header = credentials["api_key_header"] @@ -100,6 +100,11 @@ class ApiTool(Tool): headers[api_key_header] = credentials["api_key_value"] + elif credentials["auth_type"] == "api_key_query": + # For query parameter authentication, we don't add anything to headers + # The query parameter will be added in do_http_request method + pass + needed_parameters = [parameter for parameter in (self.api_bundle.parameters or []) if parameter.required] for parameter in needed_parameters: if parameter.required and parameter.name not in parameters: @@ -154,6 +159,15 @@ class ApiTool(Tool): cookies = {} files = [] + # Add API key to query parameters if auth_type is api_key_query + if self.runtime and self.runtime.credentials: + credentials = self.runtime.credentials + if credentials.get("auth_type") == "api_key_query": + api_key_query_param = credentials.get("api_key_query_param", "key") + api_key_value = credentials.get("api_key_value") + if api_key_value: + params[api_key_query_param] = api_key_value + # check parameters for parameter in self.api_bundle.openapi.get("parameters", []): value = self.get_parameter_value(parameter, parameters) @@ -213,7 +227,8 @@ class ApiTool(Tool): elif "default" in property: body[name] = property["default"] else: - body[name] = None + # omit optional parameters that weren't provided, instead of setting them to None + pass break # replace path parameters diff --git a/api/core/tools/entities/api_entities.py b/api/core/tools/entities/api_entities.py index b96c994cff..90134ba71d 100644 --- a/api/core/tools/entities/api_entities.py +++ b/api/core/tools/entities/api_entities.py @@ -1,4 +1,5 @@ -from typing import Literal, Optional +from datetime import datetime +from typing import Any, Literal, Optional from pydantic import BaseModel, Field, field_validator @@ -18,7 +19,7 @@ class ToolApiEntity(BaseModel): output_schema: Optional[dict] = None -ToolProviderTypeApiLiteral = Optional[Literal["builtin", "api", "workflow"]] +ToolProviderTypeApiLiteral = Optional[Literal["builtin", "api", "workflow", "mcp"]] class ToolProviderApiEntity(BaseModel): @@ -27,6 +28,7 @@ class ToolProviderApiEntity(BaseModel): name: str # identifier description: I18nObject icon: str | dict + icon_dark: Optional[str | dict] = Field(default=None, description="The dark icon of the tool") label: I18nObject # label type: ToolProviderType masked_credentials: Optional[dict] = None @@ -37,6 +39,10 @@ class ToolProviderApiEntity(BaseModel): plugin_unique_identifier: Optional[str] = Field(default="", description="The unique identifier of the tool") tools: list[ToolApiEntity] = Field(default_factory=list) labels: list[str] = Field(default_factory=list) + # MCP + server_url: Optional[str] = Field(default="", description="The server url of the tool") + updated_at: int = Field(default_factory=lambda: int(datetime.now().timestamp())) + server_identifier: Optional[str] = Field(default="", description="The server identifier of the MCP tool") @field_validator("tools", mode="before") @classmethod @@ -52,8 +58,13 @@ class ToolProviderApiEntity(BaseModel): for parameter in tool.get("parameters"): if parameter.get("type") == ToolParameter.ToolParameterType.SYSTEM_FILES.value: parameter["type"] = "files" + if parameter.get("input_schema") is None: + parameter.pop("input_schema", None) # ------------- - + optional_fields = self.optional_field("server_url", self.server_url) + if self.type == ToolProviderType.MCP.value: + optional_fields.update(self.optional_field("updated_at", self.updated_at)) + optional_fields.update(self.optional_field("server_identifier", self.server_identifier)) return { "id": self.id, "author": self.author, @@ -62,6 +73,7 @@ class ToolProviderApiEntity(BaseModel): "plugin_unique_identifier": self.plugin_unique_identifier, "description": self.description.to_dict(), "icon": self.icon, + "icon_dark": self.icon_dark, "label": self.label.to_dict(), "type": self.type.value, "team_credentials": self.masked_credentials, @@ -69,4 +81,9 @@ class ToolProviderApiEntity(BaseModel): "allow_delete": self.allow_delete, "tools": tools, "labels": self.labels, + **optional_fields, } + + def optional_field(self, key: str, value: Any) -> dict: + """Return dict with key-value if value is truthy, empty dict otherwise.""" + return {key: value} if value else {} diff --git a/api/core/tools/entities/tool_entities.py b/api/core/tools/entities/tool_entities.py index d2c28076ae..b5148e245f 100644 --- a/api/core/tools/entities/tool_entities.py +++ b/api/core/tools/entities/tool_entities.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_seriali from core.entities.provider_entities import ProviderConfig from core.plugin.entities.parameters import ( + MCPServerParameterType, PluginParameter, PluginParameterOption, PluginParameterType, @@ -49,6 +50,7 @@ class ToolProviderType(enum.StrEnum): API = "api" APP = "app" DATASET_RETRIEVAL = "dataset-retrieval" + MCP = "mcp" @classmethod def value_of(cls, value: str) -> "ToolProviderType": @@ -94,7 +96,8 @@ class ApiProviderAuthType(Enum): """ NONE = "none" - API_KEY = "api_key" + API_KEY_HEADER = "api_key_header" + API_KEY_QUERY = "api_key_query" @classmethod def value_of(cls, value: str) -> "ApiProviderAuthType": @@ -242,6 +245,10 @@ class ToolParameter(PluginParameter): MODEL_SELECTOR = PluginParameterType.MODEL_SELECTOR.value DYNAMIC_SELECT = PluginParameterType.DYNAMIC_SELECT.value + # MCP object and array type parameters + ARRAY = MCPServerParameterType.ARRAY.value + OBJECT = MCPServerParameterType.OBJECT.value + # deprecated, should not use. SYSTEM_FILES = PluginParameterType.SYSTEM_FILES.value @@ -260,6 +267,8 @@ class ToolParameter(PluginParameter): human_description: Optional[I18nObject] = Field(default=None, description="The description presented to the user") form: ToolParameterForm = Field(..., description="The form of the parameter, schema/form/llm") llm_description: Optional[str] = None + # MCP object and array type parameters use this field to store the schema + input_schema: Optional[dict] = None @classmethod def get_simple_instance( @@ -309,6 +318,7 @@ class ToolProviderIdentity(BaseModel): name: str = Field(..., description="The name of the tool") description: I18nObject = Field(..., description="The description of the tool") icon: str = Field(..., description="The icon of the tool") + icon_dark: Optional[str] = Field(default=None, description="The dark icon of the tool") label: I18nObject = Field(..., description="The label of the tool") tags: Optional[list[ToolLabelEnum]] = Field( default=[], diff --git a/api/core/tools/mcp_tool/provider.py b/api/core/tools/mcp_tool/provider.py new file mode 100644 index 0000000000..93f003effe --- /dev/null +++ b/api/core/tools/mcp_tool/provider.py @@ -0,0 +1,130 @@ +import json +from typing import Any + +from core.mcp.types import Tool as RemoteMCPTool +from core.tools.__base.tool_provider import ToolProviderController +from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ( + ToolDescription, + ToolEntity, + ToolIdentity, + ToolProviderEntityWithPlugin, + ToolProviderIdentity, + ToolProviderType, +) +from core.tools.mcp_tool.tool import MCPTool +from models.tools import MCPToolProvider +from services.tools.tools_transform_service import ToolTransformService + + +class MCPToolProviderController(ToolProviderController): + provider_id: str + entity: ToolProviderEntityWithPlugin + + def __init__(self, entity: ToolProviderEntityWithPlugin, provider_id: str, tenant_id: str, server_url: str) -> None: + super().__init__(entity) + self.entity = entity + self.tenant_id = tenant_id + self.provider_id = provider_id + self.server_url = server_url + + @property + def provider_type(self) -> ToolProviderType: + """ + returns the type of the provider + + :return: type of the provider + """ + return ToolProviderType.MCP + + @classmethod + def _from_db(cls, db_provider: MCPToolProvider) -> "MCPToolProviderController": + """ + from db provider + """ + tools = [] + tools_data = json.loads(db_provider.tools) + remote_mcp_tools = [RemoteMCPTool(**tool) for tool in tools_data] + user = db_provider.load_user() + tools = [ + ToolEntity( + identity=ToolIdentity( + author=user.name if user else "Anonymous", + name=remote_mcp_tool.name, + label=I18nObject(en_US=remote_mcp_tool.name, zh_Hans=remote_mcp_tool.name), + provider=db_provider.server_identifier, + icon=db_provider.icon, + ), + parameters=ToolTransformService.convert_mcp_schema_to_parameter(remote_mcp_tool.inputSchema), + description=ToolDescription( + human=I18nObject( + en_US=remote_mcp_tool.description or "", zh_Hans=remote_mcp_tool.description or "" + ), + llm=remote_mcp_tool.description or "", + ), + output_schema=None, + has_runtime_parameters=len(remote_mcp_tool.inputSchema) > 0, + ) + for remote_mcp_tool in remote_mcp_tools + ] + + return cls( + entity=ToolProviderEntityWithPlugin( + identity=ToolProviderIdentity( + author=user.name if user else "Anonymous", + name=db_provider.name, + label=I18nObject(en_US=db_provider.name, zh_Hans=db_provider.name), + description=I18nObject(en_US="", zh_Hans=""), + icon=db_provider.icon, + ), + plugin_id=None, + credentials_schema=[], + tools=tools, + ), + provider_id=db_provider.server_identifier or "", + tenant_id=db_provider.tenant_id or "", + server_url=db_provider.decrypted_server_url, + ) + + def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None: + """ + validate the credentials of the provider + """ + pass + + def get_tool(self, tool_name: str) -> MCPTool: # type: ignore + """ + return tool with given name + """ + tool_entity = next( + (tool_entity for tool_entity in self.entity.tools if tool_entity.identity.name == tool_name), None + ) + + if not tool_entity: + raise ValueError(f"Tool with name {tool_name} not found") + + return MCPTool( + entity=tool_entity, + runtime=ToolRuntime(tenant_id=self.tenant_id), + tenant_id=self.tenant_id, + icon=self.entity.identity.icon, + server_url=self.server_url, + provider_id=self.provider_id, + ) + + def get_tools(self) -> list[MCPTool]: # type: ignore + """ + get all tools + """ + return [ + MCPTool( + entity=tool_entity, + runtime=ToolRuntime(tenant_id=self.tenant_id), + tenant_id=self.tenant_id, + icon=self.entity.identity.icon, + server_url=self.server_url, + provider_id=self.provider_id, + ) + for tool_entity in self.entity.tools + ] diff --git a/api/core/tools/mcp_tool/tool.py b/api/core/tools/mcp_tool/tool.py new file mode 100644 index 0000000000..d1bacbc735 --- /dev/null +++ b/api/core/tools/mcp_tool/tool.py @@ -0,0 +1,92 @@ +import base64 +import json +from collections.abc import Generator +from typing import Any, Optional + +from core.mcp.error import MCPAuthError, MCPConnectionError +from core.mcp.mcp_client import MCPClient +from core.mcp.types import ImageContent, TextContent +from core.tools.__base.tool import Tool +from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolParameter, ToolProviderType + + +class MCPTool(Tool): + tenant_id: str + icon: str + runtime_parameters: Optional[list[ToolParameter]] + server_url: str + provider_id: str + + def __init__( + self, entity: ToolEntity, runtime: ToolRuntime, tenant_id: str, icon: str, server_url: str, provider_id: str + ) -> None: + super().__init__(entity, runtime) + self.tenant_id = tenant_id + self.icon = icon + self.runtime_parameters = None + self.server_url = server_url + self.provider_id = provider_id + + def tool_provider_type(self) -> ToolProviderType: + return ToolProviderType.MCP + + def _invoke( + self, + user_id: str, + tool_parameters: dict[str, Any], + conversation_id: Optional[str] = None, + app_id: Optional[str] = None, + message_id: Optional[str] = None, + ) -> Generator[ToolInvokeMessage, None, None]: + from core.tools.errors import ToolInvokeError + + try: + with MCPClient(self.server_url, self.provider_id, self.tenant_id, authed=True) as mcp_client: + tool_parameters = self._handle_none_parameter(tool_parameters) + result = mcp_client.invoke_tool(tool_name=self.entity.identity.name, tool_args=tool_parameters) + except MCPAuthError as e: + raise ToolInvokeError("Please auth the tool first") from e + except MCPConnectionError as e: + raise ToolInvokeError(f"Failed to connect to MCP server: {e}") from e + except Exception as e: + raise ToolInvokeError(f"Failed to invoke tool: {e}") from e + + for content in result.content: + if isinstance(content, TextContent): + try: + content_json = json.loads(content.text) + if isinstance(content_json, dict): + yield self.create_json_message(content_json) + elif isinstance(content_json, list): + for item in content_json: + yield self.create_json_message(item) + else: + yield self.create_text_message(content.text) + except json.JSONDecodeError: + yield self.create_text_message(content.text) + + elif isinstance(content, ImageContent): + yield self.create_blob_message( + blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType} + ) + + def fork_tool_runtime(self, runtime: ToolRuntime) -> "MCPTool": + return MCPTool( + entity=self.entity, + runtime=runtime, + tenant_id=self.tenant_id, + icon=self.icon, + server_url=self.server_url, + provider_id=self.provider_id, + ) + + def _handle_none_parameter(self, parameter: dict[str, Any]) -> dict[str, Any]: + """ + in mcp tool invoke, if the parameter is empty, it will be set to None + """ + return { + key: value + for key, value in parameter.items() + if value is not None and not (isinstance(value, str) and value.strip() == "") + } diff --git a/api/core/tools/signature.py b/api/core/tools/signature.py index e80005d7bf..5cdf473542 100644 --- a/api/core/tools/signature.py +++ b/api/core/tools/signature.py @@ -9,9 +9,10 @@ from configs import dify_config def sign_tool_file(tool_file_id: str, extension: str) -> str: """ - sign file to get a temporary url + sign file to get a temporary url for plugin access """ - base_url = dify_config.FILES_URL + # Use internal URL for plugin/tool file access in Docker environments + base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL file_preview_url = f"{base_url}/files/tools/{tool_file_id}{extension}" timestamp = str(int(time.time())) diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index b849f51064..ece02f9d59 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -35,9 +35,10 @@ class ToolFileManager: @staticmethod def sign_file(tool_file_id: str, extension: str) -> str: """ - sign file to get a temporary url + sign file to get a temporary url for plugin access """ - base_url = dify_config.FILES_URL + # Use internal URL for plugin/tool file access in Docker environments + base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL file_preview_url = f"{base_url}/files/tools/{tool_file_id}{extension}" timestamp = str(int(time.time())) diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index 0bfe6329b1..22a9853b41 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -4,7 +4,7 @@ import mimetypes from collections.abc import Generator from os import listdir, path from threading import Lock -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast from yarl import URL @@ -13,9 +13,13 @@ from core.plugin.entities.plugin import ToolProviderID from core.plugin.impl.tool import PluginToolManager from core.tools.__base.tool_provider import ToolProviderController from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.mcp_tool.provider import MCPToolProviderController +from core.tools.mcp_tool.tool import MCPTool from core.tools.plugin_tool.provider import PluginToolProviderController from core.tools.plugin_tool.tool import PluginTool from core.tools.workflow_as_tool.provider import WorkflowToolProviderController +from core.workflow.entities.variable_pool import VariablePool +from services.tools.mcp_tools_mange_service import MCPToolManageService if TYPE_CHECKING: from core.workflow.nodes.tool.entities import ToolEntity @@ -49,7 +53,7 @@ from core.tools.utils.configuration import ( ) from core.tools.workflow_as_tool.tool import WorkflowTool from extensions.ext_database import db -from models.tools import ApiToolProvider, BuiltinToolProvider, WorkflowToolProvider +from models.tools import ApiToolProvider, BuiltinToolProvider, MCPToolProvider, WorkflowToolProvider from services.tools.tools_transform_service import ToolTransformService logger = logging.getLogger(__name__) @@ -156,7 +160,7 @@ class ToolManager: tenant_id: str, invoke_from: InvokeFrom = InvokeFrom.DEBUGGER, tool_invoke_from: ToolInvokeFrom = ToolInvokeFrom.AGENT, - ) -> Union[BuiltinTool, PluginTool, ApiTool, WorkflowTool]: + ) -> Union[BuiltinTool, PluginTool, ApiTool, WorkflowTool, MCPTool]: """ get the tool runtime @@ -292,6 +296,8 @@ class ToolManager: raise NotImplementedError("app provider not implemented") elif provider_type == ToolProviderType.PLUGIN: return cls.get_plugin_provider(provider_id, tenant_id).get_tool(tool_name) + elif provider_type == ToolProviderType.MCP: + return cls.get_mcp_provider_controller(tenant_id, provider_id).get_tool(tool_name) else: raise ToolProviderNotFoundError(f"provider type {provider_type.value} not found") @@ -302,6 +308,7 @@ class ToolManager: app_id: str, agent_tool: AgentToolEntity, invoke_from: InvokeFrom = InvokeFrom.DEBUGGER, + variable_pool: Optional[VariablePool] = None, ) -> Tool: """ get the agent tool runtime @@ -316,24 +323,9 @@ class ToolManager: ) runtime_parameters = {} parameters = tool_entity.get_merged_runtime_parameters() - for parameter in parameters: - # check file types - if ( - parameter.type - in { - ToolParameter.ToolParameterType.SYSTEM_FILES, - ToolParameter.ToolParameterType.FILE, - ToolParameter.ToolParameterType.FILES, - } - and parameter.required - ): - raise ValueError(f"file type parameter {parameter.name} not supported in agent") - - if parameter.form == ToolParameter.ToolParameterForm.FORM: - # save tool parameter to tool entity memory - value = parameter.init_frontend_parameter(agent_tool.tool_parameters.get(parameter.name)) - runtime_parameters[parameter.name] = value - + runtime_parameters = cls._convert_tool_parameters_type( + parameters, variable_pool, agent_tool.tool_parameters, typ="agent" + ) # decrypt runtime parameters encryption_manager = ToolParameterConfigurationManager( tenant_id=tenant_id, @@ -357,10 +349,12 @@ class ToolManager: node_id: str, workflow_tool: "ToolEntity", invoke_from: InvokeFrom = InvokeFrom.DEBUGGER, + variable_pool: Optional[VariablePool] = None, ) -> Tool: """ get the workflow tool runtime """ + tool_runtime = cls.get_tool_runtime( provider_type=workflow_tool.provider_type, provider_id=workflow_tool.provider_id, @@ -369,15 +363,11 @@ class ToolManager: invoke_from=invoke_from, tool_invoke_from=ToolInvokeFrom.WORKFLOW, ) - runtime_parameters = {} - parameters = tool_runtime.get_merged_runtime_parameters() - - for parameter in parameters: - # save tool parameter to tool entity memory - if parameter.form == ToolParameter.ToolParameterForm.FORM: - value = parameter.init_frontend_parameter(workflow_tool.tool_configurations.get(parameter.name)) - runtime_parameters[parameter.name] = value + parameters = tool_runtime.get_merged_runtime_parameters() + runtime_parameters = cls._convert_tool_parameters_type( + parameters, variable_pool, workflow_tool.tool_configurations, typ="workflow" + ) # decrypt runtime parameters encryption_manager = ToolParameterConfigurationManager( tenant_id=tenant_id, @@ -569,7 +559,7 @@ class ToolManager: filters = [] if not typ: - filters.extend(["builtin", "api", "workflow"]) + filters.extend(["builtin", "api", "workflow", "mcp"]) else: filters.append(typ) @@ -663,6 +653,10 @@ class ToolManager: labels=labels.get(provider_controller.provider_id, []), ) result_providers[f"workflow_provider.{user_provider.name}"] = user_provider + if "mcp" in filters: + mcp_providers = MCPToolManageService.retrieve_mcp_tools(tenant_id, for_list=True) + for mcp_provider in mcp_providers: + result_providers[f"mcp_provider.{mcp_provider.name}"] = mcp_provider return BuiltinToolProviderSort.sort(list(result_providers.values())) @@ -690,14 +684,47 @@ class ToolManager: if provider is None: raise ToolProviderNotFoundError(f"api provider {provider_id} not found") + auth_type = ApiProviderAuthType.NONE + provider_auth_type = provider.credentials.get("auth_type") + if provider_auth_type in ("api_key_header", "api_key"): # backward compatibility + auth_type = ApiProviderAuthType.API_KEY_HEADER + elif provider_auth_type == "api_key_query": + auth_type = ApiProviderAuthType.API_KEY_QUERY + controller = ApiToolProviderController.from_db( provider, - ApiProviderAuthType.API_KEY if provider.credentials["auth_type"] == "api_key" else ApiProviderAuthType.NONE, + auth_type, ) controller.load_bundled_tools(provider.tools) return controller, provider.credentials + @classmethod + def get_mcp_provider_controller(cls, tenant_id: str, provider_id: str) -> MCPToolProviderController: + """ + get the api provider + + :param tenant_id: the id of the tenant + :param provider_id: the id of the provider + + :return: the provider controller, the credentials + """ + provider: MCPToolProvider | None = ( + db.session.query(MCPToolProvider) + .filter( + MCPToolProvider.server_identifier == provider_id, + MCPToolProvider.tenant_id == tenant_id, + ) + .first() + ) + + if provider is None: + raise ToolProviderNotFoundError(f"mcp provider {provider_id} not found") + + controller = MCPToolProviderController._from_db(provider) + + return controller + @classmethod def user_get_api_provider(cls, provider: str, tenant_id: str) -> dict: """ @@ -725,9 +752,16 @@ class ToolManager: credentials = {} # package tool provider controller + auth_type = ApiProviderAuthType.NONE + credentials_auth_type = credentials.get("auth_type") + if credentials_auth_type in ("api_key_header", "api_key"): # backward compatibility + auth_type = ApiProviderAuthType.API_KEY_HEADER + elif credentials_auth_type == "api_key_query": + auth_type = ApiProviderAuthType.API_KEY_QUERY + controller = ApiToolProviderController.from_db( provider_obj, - ApiProviderAuthType.API_KEY if credentials["auth_type"] == "api_key" else ApiProviderAuthType.NONE, + auth_type, ) # init tool configuration tool_configuration = ProviderConfigEncrypter( @@ -826,6 +860,22 @@ class ToolManager: except Exception: return {"background": "#252525", "content": "\ud83d\ude01"} + @classmethod + def generate_mcp_tool_icon_url(cls, tenant_id: str, provider_id: str) -> dict[str, str] | str: + try: + mcp_provider: MCPToolProvider | None = ( + db.session.query(MCPToolProvider) + .filter(MCPToolProvider.tenant_id == tenant_id, MCPToolProvider.server_identifier == provider_id) + .first() + ) + + if mcp_provider is None: + raise ToolProviderNotFoundError(f"mcp provider {provider_id} not found") + + return mcp_provider.provider_icon + except Exception: + return {"background": "#252525", "content": "\ud83d\ude01"} + @classmethod def get_tool_icon( cls, @@ -863,8 +913,61 @@ class ToolManager: except Exception: return {"background": "#252525", "content": "\ud83d\ude01"} raise ValueError(f"plugin provider {provider_id} not found") + elif provider_type == ToolProviderType.MCP: + return cls.generate_mcp_tool_icon_url(tenant_id, provider_id) else: raise ValueError(f"provider type {provider_type} not found") + @classmethod + def _convert_tool_parameters_type( + cls, + parameters: list[ToolParameter], + variable_pool: Optional[VariablePool], + tool_configurations: dict[str, Any], + typ: Literal["agent", "workflow", "tool"] = "workflow", + ) -> dict[str, Any]: + """ + Convert tool parameters type + """ + from core.workflow.nodes.tool.entities import ToolNodeData + from core.workflow.nodes.tool.exc import ToolParameterError + + runtime_parameters = {} + for parameter in parameters: + if ( + parameter.type + in { + ToolParameter.ToolParameterType.SYSTEM_FILES, + ToolParameter.ToolParameterType.FILE, + ToolParameter.ToolParameterType.FILES, + } + and parameter.required + and typ == "agent" + ): + raise ValueError(f"file type parameter {parameter.name} not supported in agent") + # save tool parameter to tool entity memory + if parameter.form == ToolParameter.ToolParameterForm.FORM: + if variable_pool: + config = tool_configurations.get(parameter.name, {}) + if not (config and isinstance(config, dict) and config.get("value") is not None): + continue + tool_input = ToolNodeData.ToolInput(**tool_configurations.get(parameter.name, {})) + if tool_input.type == "variable": + variable = variable_pool.get(tool_input.value) + if variable is None: + raise ToolParameterError(f"Variable {tool_input.value} does not exist") + parameter_value = variable.value + elif tool_input.type in {"mixed", "constant"}: + segment_group = variable_pool.convert_template(str(tool_input.value)) + parameter_value = segment_group.text + else: + raise ToolParameterError(f"Unknown tool input type '{tool_input.type}'") + runtime_parameters[parameter.name] = parameter_value + + else: + value = parameter.init_frontend_parameter(tool_configurations.get(parameter.name)) + runtime_parameters[parameter.name] = value + return runtime_parameters + ToolManager.load_hardcoded_providers_cache() diff --git a/api/core/tools/utils/configuration.py b/api/core/tools/utils/configuration.py index 1f23e90351..251fedf56e 100644 --- a/api/core/tools/utils/configuration.py +++ b/api/core/tools/utils/configuration.py @@ -72,21 +72,21 @@ class ProviderConfigEncrypter(BaseModel): return data - def decrypt(self, data: dict[str, str]) -> dict[str, str]: + def decrypt(self, data: dict[str, str], use_cache: bool = True) -> dict[str, str]: """ decrypt tool credentials with tenant id return a deep copy of credentials with decrypted values """ - cache = ToolProviderCredentialsCache( - tenant_id=self.tenant_id, - identity_id=f"{self.provider_type}.{self.provider_identity}", - cache_type=ToolProviderCredentialsCacheType.PROVIDER, - ) - cached_credentials = cache.get() - if cached_credentials: - return cached_credentials - + if use_cache: + cache = ToolProviderCredentialsCache( + tenant_id=self.tenant_id, + identity_id=f"{self.provider_type}.{self.provider_identity}", + cache_type=ToolProviderCredentialsCacheType.PROVIDER, + ) + cached_credentials = cache.get() + if cached_credentials: + return cached_credentials data = self._deep_copy(data) # get fields need to be decrypted fields = dict[str, BasicProviderConfig]() @@ -105,7 +105,8 @@ class ProviderConfigEncrypter(BaseModel): except Exception: pass - cache.set(data) + if use_cache: + cache.set(data) return data def delete_tool_credentials_cache(self): diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index 3f844e8234..a3c84615ca 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -1,5 +1,4 @@ import re -import uuid from json import dumps as json_dumps from json import loads as json_loads from json.decoder import JSONDecodeError @@ -154,7 +153,7 @@ class ApiBasedToolSchemaParser: # remove special characters like / to ensure the operation id is valid ^[a-zA-Z0-9_-]{1,64}$ path = re.sub(r"[^a-zA-Z0-9_-]", "", path) if not path: - path = str(uuid.uuid4()) + path = "" interface["operation"]["operationId"] = f"{path}_{interface['method']}" diff --git a/api/core/tools/workflow_as_tool/tool.py b/api/core/tools/workflow_as_tool/tool.py index 57c93d1d45..10bf8ca640 100644 --- a/api/core/tools/workflow_as_tool/tool.py +++ b/api/core/tools/workflow_as_tool/tool.py @@ -8,7 +8,12 @@ from flask_login import current_user from core.file import FILE_MODEL_IDENTITY, File, FileTransferMethod from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime -from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolParameter, ToolProviderType +from core.tools.entities.tool_entities import ( + ToolEntity, + ToolInvokeMessage, + ToolParameter, + ToolProviderType, +) from core.tools.errors import ToolInvokeError from extensions.ext_database import db from factories.file_factory import build_from_mapping diff --git a/api/core/workflow/callbacks/workflow_logging_callback.py b/api/core/workflow/callbacks/workflow_logging_callback.py index e6813a3997..12b5203ca3 100644 --- a/api/core/workflow/callbacks/workflow_logging_callback.py +++ b/api/core/workflow/callbacks/workflow_logging_callback.py @@ -232,14 +232,14 @@ class WorkflowLoggingCallback(WorkflowCallback): Publish loop started """ self.print_text("\n[LoopRunStartedEvent]", color="blue") - self.print_text(f"Loop Node ID: {event.loop_id}", color="blue") + self.print_text(f"Loop Node ID: {event.loop_node_id}", color="blue") def on_workflow_loop_next(self, event: LoopRunNextEvent) -> None: """ Publish loop next """ self.print_text("\n[LoopRunNextEvent]", color="blue") - self.print_text(f"Loop Node ID: {event.loop_id}", color="blue") + self.print_text(f"Loop Node ID: {event.loop_node_id}", color="blue") self.print_text(f"Loop Index: {event.index}", color="blue") def on_workflow_loop_completed(self, event: LoopRunSucceededEvent | LoopRunFailedEvent) -> None: @@ -250,7 +250,7 @@ class WorkflowLoggingCallback(WorkflowCallback): "\n[LoopRunSucceededEvent]" if isinstance(event, LoopRunSucceededEvent) else "\n[LoopRunFailedEvent]", color="blue", ) - self.print_text(f"Node ID: {event.loop_id}", color="blue") + self.print_text(f"Loop Node ID: {event.loop_node_id}", color="blue") def print_text(self, text: str, color: Optional[str] = None, end: str = "\n") -> None: """Print text with highlighting and no end characters.""" diff --git a/api/core/workflow/graph_engine/entities/graph.py b/api/core/workflow/graph_engine/entities/graph.py index 8e5b1e7142..362777a199 100644 --- a/api/core/workflow/graph_engine/entities/graph.py +++ b/api/core/workflow/graph_engine/entities/graph.py @@ -334,7 +334,7 @@ class Graph(BaseModel): parallel = GraphParallel( start_from_node_id=start_node_id, - parent_parallel_id=parent_parallel.id if parent_parallel else None, + parent_parallel_id=parent_parallel_id, parent_parallel_start_node_id=parent_parallel.start_from_node_id if parent_parallel else None, ) parallel_mapping[parallel.id] = parallel diff --git a/api/core/workflow/nodes/agent/agent_node.py b/api/core/workflow/nodes/agent/agent_node.py index 766cdb604f..678b99d546 100644 --- a/api/core/workflow/nodes/agent/agent_node.py +++ b/api/core/workflow/nodes/agent/agent_node.py @@ -3,11 +3,13 @@ import uuid from collections.abc import Generator, Mapping, Sequence from typing import Any, Optional, cast +from packaging.version import Version from sqlalchemy import select from sqlalchemy.orm import Session from core.agent.entities import AgentToolEntity from core.agent.plugin_entities import AgentStrategyParameter +from core.agent.strategy.plugin import PluginAgentStrategy from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance, ModelManager from core.model_runtime.entities.model_entities import AIModelEntity, ModelType @@ -73,12 +75,14 @@ class AgentNode(ToolNode): agent_parameters=agent_parameters, variable_pool=self.graph_runtime_state.variable_pool, node_data=node_data, + strategy=strategy, ) parameters_for_log = self._generate_agent_parameters( agent_parameters=agent_parameters, variable_pool=self.graph_runtime_state.variable_pool, node_data=node_data, for_log=True, + strategy=strategy, ) # get conversation id @@ -155,6 +159,7 @@ class AgentNode(ToolNode): variable_pool: VariablePool, node_data: AgentNodeData, for_log: bool = False, + strategy: PluginAgentStrategy, ) -> dict[str, Any]: """ Generate parameters based on the given tool parameters, variable pool, and node data. @@ -207,7 +212,7 @@ class AgentNode(ToolNode): if parameter.type == "array[tools]": value = cast(list[dict[str, Any]], value) value = [tool for tool in value if tool.get("enabled", False)] - + value = self._filter_mcp_type_tool(strategy, value) for tool in value: if "schemas" in tool: tool.pop("schemas") @@ -244,9 +249,9 @@ class AgentNode(ToolNode): ) extra = tool.get("extra", {}) - + runtime_variable_pool = variable_pool if self.node_data.version != "1" else None tool_runtime = ToolManager.get_agent_tool_runtime( - self.tenant_id, self.app_id, entity, self.invoke_from + self.tenant_id, self.app_id, entity, self.invoke_from, runtime_variable_pool ) if tool_runtime.entity.description: tool_runtime.entity.description.llm = ( @@ -398,3 +403,16 @@ class AgentNode(ToolNode): except ValueError: model_schema.features.remove(feature) return model_schema + + def _filter_mcp_type_tool(self, strategy: PluginAgentStrategy, tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + """ + Filter MCP type tool + :param strategy: plugin agent strategy + :param tool: tool + :return: filtered tool dict + """ + meta_version = strategy.meta_version + if meta_version and Version(meta_version) > Version("0.0.1"): + return tools + else: + return [tool for tool in tools if tool.get("type") != ToolProviderType.MCP.value] diff --git a/api/core/workflow/nodes/iteration/iteration_node.py b/api/core/workflow/nodes/iteration/iteration_node.py index c447f433aa..8b566c83cd 100644 --- a/api/core/workflow/nodes/iteration/iteration_node.py +++ b/api/core/workflow/nodes/iteration/iteration_node.py @@ -521,18 +521,52 @@ class IterationNode(BaseNode[IterationNodeData]): ) return elif self.node_data.error_handle_mode == ErrorHandleMode.TERMINATED: - yield IterationRunFailedEvent( - iteration_id=self.id, - iteration_node_id=self.node_id, - iteration_node_type=self.node_type, - iteration_node_data=self.node_data, - start_at=start_at, - inputs=inputs, - outputs={"output": None}, - steps=len(iterator_list_value), - metadata={"total_tokens": graph_engine.graph_runtime_state.total_tokens}, - error=event.error, + yield NodeInIterationFailedEvent( + **metadata_event.model_dump(), ) + outputs[current_index] = None + + # clean nodes resources + for node_id in iteration_graph.node_ids: + variable_pool.remove([node_id]) + + # iteration run failed + if self.node_data.is_parallel: + yield IterationRunFailedEvent( + iteration_id=self.id, + iteration_node_id=self.node_id, + iteration_node_type=self.node_type, + iteration_node_data=self.node_data, + parallel_mode_run_id=parallel_mode_run_id, + start_at=start_at, + inputs=inputs, + outputs={"output": outputs}, + steps=len(iterator_list_value), + metadata={"total_tokens": graph_engine.graph_runtime_state.total_tokens}, + error=event.error, + ) + else: + yield IterationRunFailedEvent( + iteration_id=self.id, + iteration_node_id=self.node_id, + iteration_node_type=self.node_type, + iteration_node_data=self.node_data, + start_at=start_at, + inputs=inputs, + outputs={"output": outputs}, + steps=len(iterator_list_value), + metadata={"total_tokens": graph_engine.graph_runtime_state.total_tokens}, + error=event.error, + ) + + # stop the iterator + yield RunCompletedEvent( + run_result=NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + error=event.error, + ) + ) + return yield metadata_event current_output_segment = variable_pool.get(self.node_data.output_selector) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index b34d62d669..f05d93d83e 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -144,6 +144,8 @@ class KnowledgeRetrievalNode(LLMNode): error=str(e), error_type=type(e).__name__, ) + finally: + db.session.close() def _fetch_dataset_retriever(self, node_data: KnowledgeRetrievalNodeData, query: str) -> list[dict[str, Any]]: available_datasets = [] @@ -171,6 +173,9 @@ class KnowledgeRetrievalNode(LLMNode): .all() ) + # avoid blocking at retrieval + db.session.close() + for dataset in results: # pass if dataset is not available if not dataset: diff --git a/api/core/workflow/nodes/node_mapping.py b/api/core/workflow/nodes/node_mapping.py index 67cc884f20..ccfaec4a8c 100644 --- a/api/core/workflow/nodes/node_mapping.py +++ b/api/core/workflow/nodes/node_mapping.py @@ -73,6 +73,7 @@ NODE_TYPE_CLASSES_MAPPING: Mapping[NodeType, Mapping[str, type[BaseNode]]] = { }, NodeType.TOOL: { LATEST_VERSION: ToolNode, + "2": ToolNode, "1": ToolNode, }, NodeType.VARIABLE_AGGREGATOR: { @@ -122,6 +123,7 @@ NODE_TYPE_CLASSES_MAPPING: Mapping[NodeType, Mapping[str, type[BaseNode]]] = { }, NodeType.AGENT: { LATEST_VERSION: AgentNode, + "2": AgentNode, "1": AgentNode, }, } diff --git a/api/core/workflow/nodes/tool/entities.py b/api/core/workflow/nodes/tool/entities.py index 21023d4ab7..691f6e0196 100644 --- a/api/core/workflow/nodes/tool/entities.py +++ b/api/core/workflow/nodes/tool/entities.py @@ -41,6 +41,10 @@ class ToolNodeData(BaseNodeData, ToolEntity): def check_type(cls, value, validation_info: ValidationInfo): typ = value value = validation_info.data.get("value") + + if value is None: + return typ + if typ == "mixed" and not isinstance(value, str): raise ValueError("value must be a string") elif typ == "variable": @@ -54,3 +58,22 @@ class ToolNodeData(BaseNodeData, ToolEntity): return typ tool_parameters: dict[str, ToolInput] + + @field_validator("tool_parameters", mode="before") + @classmethod + def filter_none_tool_inputs(cls, value): + if not isinstance(value, dict): + return value + + return { + key: tool_input + for key, tool_input in value.items() + if tool_input is not None and cls._has_valid_value(tool_input) + } + + @staticmethod + def _has_valid_value(tool_input): + """Check if the value is valid""" + if isinstance(tool_input, dict): + return tool_input.get("value") is not None + return getattr(tool_input, "value", None) is not None diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 59b3b1e2ae..48627a229d 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -67,8 +67,9 @@ class ToolNode(BaseNode[ToolNodeData]): try: from core.tools.tool_manager import ToolManager + variable_pool = self.graph_runtime_state.variable_pool if self.node_data.version != "1" else None tool_runtime = ToolManager.get_workflow_tool_runtime( - self.tenant_id, self.app_id, self.node_id, self.node_data, self.invoke_from + self.tenant_id, self.app_id, self.node_id, self.node_data, self.invoke_from, variable_pool ) except ToolNodeError as e: yield RunCompletedEvent( @@ -95,7 +96,6 @@ class ToolNode(BaseNode[ToolNodeData]): node_data=self.node_data, for_log=True, ) - # get conversation id conversation_id = self.graph_runtime_state.variable_pool.get(["sys", SystemVariableKey.CONVERSATION_ID]) @@ -285,7 +285,8 @@ class ToolNode(BaseNode[ToolNodeData]): for key, value in msg_metadata.items() if key in WorkflowNodeExecutionMetadataKey.__members__.values() } - json.append(message.message.json_object) + if message.message.json_object is not None: + json.append(message.message.json_object) elif message.type == ToolInvokeMessage.MessageType.LINK: assert isinstance(message.message, ToolInvokeMessage.TextMessage) stream_text = f"Link: {message.message.text}\n" @@ -328,6 +329,7 @@ class ToolNode(BaseNode[ToolNodeData]): icon = current_plugin.declaration.icon except StopIteration: pass + icon_dark = None try: builtin_tool = next( provider @@ -338,10 +340,12 @@ class ToolNode(BaseNode[ToolNodeData]): if provider.name == dict_metadata["provider"] ) icon = builtin_tool.icon + icon_dark = builtin_tool.icon_dark except StopIteration: pass dict_metadata["icon"] = icon + dict_metadata["icon_dark"] = icon_dark message.message.metadata = dict_metadata agent_log = AgentLogEvent( id=message.message.id, @@ -369,31 +373,31 @@ class ToolNode(BaseNode[ToolNodeData]): agent_logs.append(agent_log) yield agent_log + # Add agent_logs to outputs['json'] to ensure frontend can access thinking process - json_output: dict[str, Any] = {} - if json: - if isinstance(json, list) and len(json) == 1: - # If json is a list with only one element, convert it to a dictionary - json_output = json[0] if isinstance(json[0], dict) else {"data": json[0]} - elif isinstance(json, list): - # If json is a list with multiple elements, create a dictionary containing all data - json_output = {"data": json} + json_output: list[dict[str, Any]] = [] + # Step 1: append each agent log as its own dict. if agent_logs: - # Add agent_logs to json output - json_output["agent_logs"] = [ - { - "id": log.id, - "parent_id": log.parent_id, - "error": log.error, - "status": log.status, - "data": log.data, - "label": log.label, - "metadata": log.metadata, - "node_id": log.node_id, - } - for log in agent_logs - ] + for log in agent_logs: + json_output.append( + { + "id": log.id, + "parent_id": log.parent_id, + "error": log.error, + "status": log.status, + "data": log.data, + "label": log.label, + "metadata": log.metadata, + "node_id": log.node_id, + } + ) + # Step 2: normalize JSON into {"data": [...]}.change json to list[dict] + if json: + json_output.extend(json) + else: + json_output.append({"data": []}) + yield RunCompletedEvent( run_result=NodeRunResult( status=WorkflowNodeExecutionStatus.SUCCEEDED, diff --git a/api/extensions/ext_blueprints.py b/api/extensions/ext_blueprints.py index 316be12f5c..a4d013ffc0 100644 --- a/api/extensions/ext_blueprints.py +++ b/api/extensions/ext_blueprints.py @@ -10,6 +10,7 @@ def init_app(app: DifyApp): from controllers.console import bp as console_app_bp from controllers.files import bp as files_bp from controllers.inner_api import bp as inner_api_bp + from controllers.mcp import bp as mcp_bp from controllers.service_api import bp as service_api_bp from controllers.web import bp as web_bp @@ -46,3 +47,4 @@ def init_app(app: DifyApp): app.register_blueprint(files_bp) app.register_blueprint(inner_api_bp) + app.register_blueprint(mcp_bp) diff --git a/api/extensions/ext_login.py b/api/extensions/ext_login.py index 3b4d787d01..11d1856ac4 100644 --- a/api/extensions/ext_login.py +++ b/api/extensions/ext_login.py @@ -10,7 +10,7 @@ from dify_app import DifyApp from extensions.ext_database import db from libs.passport import PassportService from models.account import Account, Tenant, TenantAccountJoin -from models.model import EndUser +from models.model import AppMCPServer, EndUser from services.account_service import AccountService login_manager = flask_login.LoginManager() @@ -74,6 +74,21 @@ def load_user_from_request(request_from_flask_login): if not end_user: raise NotFound("End user not found.") return end_user + elif request.blueprint == "mcp": + server_code = request.view_args.get("server_code") if request.view_args else None + if not server_code: + raise Unauthorized("Invalid Authorization token.") + app_mcp_server = db.session.query(AppMCPServer).filter(AppMCPServer.server_code == server_code).first() + if not app_mcp_server: + raise NotFound("App MCP server not found.") + end_user = ( + db.session.query(EndUser) + .filter(EndUser.external_user_id == app_mcp_server.id, EndUser.type == "mcp") + .first() + ) + if not end_user: + raise NotFound("End user not found.") + return end_user @user_logged_in.connect diff --git a/api/extensions/ext_otel.py b/api/extensions/ext_otel.py index 23cf4c5cab..b62b0b60d6 100644 --- a/api/extensions/ext_otel.py +++ b/api/extensions/ext_otel.py @@ -12,6 +12,7 @@ from flask_login import user_loaded_from_request, user_logged_in # type: ignore from configs import dify_config from dify_app import DifyApp +from libs.helper import extract_tenant_id from models import Account, EndUser @@ -24,11 +25,8 @@ def on_user_loaded(_sender, user: Union["Account", "EndUser"]): if user: try: current_span = get_current_span() - if isinstance(user, Account) and user.current_tenant_id: - tenant_id = user.current_tenant_id - elif isinstance(user, EndUser): - tenant_id = user.tenant_id - else: + tenant_id = extract_tenant_id(user) + if not tenant_id: return if current_span: current_span.set_attribute("service.tenant.id", tenant_id) diff --git a/api/extensions/ext_redis.py b/api/extensions/ext_redis.py index c283b1b7ca..be2f6115f7 100644 --- a/api/extensions/ext_redis.py +++ b/api/extensions/ext_redis.py @@ -1,6 +1,10 @@ +import functools +import logging +from collections.abc import Callable from typing import Any, Union import redis +from redis import RedisError from redis.cache import CacheConfig from redis.cluster import ClusterNode, RedisCluster from redis.connection import Connection, SSLConnection @@ -9,6 +13,8 @@ from redis.sentinel import Sentinel from configs import dify_config from dify_app import DifyApp +logger = logging.getLogger(__name__) + class RedisClientWrapper: """ @@ -115,3 +121,25 @@ def init_app(app: DifyApp): redis_client.initialize(redis.Redis(connection_pool=pool)) app.extensions["redis"] = redis_client + + +def redis_fallback(default_return: Any = None): + """ + decorator to handle Redis operation exceptions and return a default value when Redis is unavailable. + + Args: + default_return: The value to return when a Redis operation fails. Defaults to None. + """ + + def decorator(func: Callable): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except RedisError as e: + logger.warning(f"Redis operation failed in {func.__name__}: {str(e)}", exc_info=True) + return default_return + + return wrapper + + return decorator diff --git a/api/factories/agent_factory.py b/api/factories/agent_factory.py index 4b12afb528..2570bc22f1 100644 --- a/api/factories/agent_factory.py +++ b/api/factories/agent_factory.py @@ -10,6 +10,6 @@ def get_plugin_agent_strategy( agent_provider = manager.fetch_agent_strategy_provider(tenant_id, agent_strategy_provider_name) for agent_strategy in agent_provider.declaration.strategies: if agent_strategy.identity.name == agent_strategy_name: - return PluginAgentStrategy(tenant_id, agent_strategy) + return PluginAgentStrategy(tenant_id, agent_strategy, agent_provider.meta.version) raise ValueError(f"Agent strategy {agent_strategy_name} not found") diff --git a/api/fields/app_fields.py b/api/fields/app_fields.py index 500ca47c7e..73c224542a 100644 --- a/api/fields/app_fields.py +++ b/api/fields/app_fields.py @@ -1,8 +1,21 @@ +import json + from flask_restful import fields from fields.workflow_fields import workflow_partial_fields from libs.helper import AppIconUrlField, TimestampField + +class JsonStringField(fields.Raw): + def format(self, value): + if isinstance(value, str): + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return value + + app_detail_kernel_fields = { "id": fields.String, "name": fields.String, @@ -218,3 +231,14 @@ app_import_fields = { app_import_check_dependencies_fields = { "leaked_dependencies": fields.List(fields.Nested(leaked_dependency_fields)), } + +app_server_fields = { + "id": fields.String, + "name": fields.String, + "server_code": fields.String, + "description": fields.String, + "status": fields.String, + "parameters": JsonStringField, + "created_at": TimestampField, + "updated_at": TimestampField, +} diff --git a/api/fields/workflow_fields.py b/api/fields/workflow_fields.py index 9f1bef3b36..f00ea71c54 100644 --- a/api/fields/workflow_fields.py +++ b/api/fields/workflow_fields.py @@ -17,6 +17,7 @@ class EnvironmentVariableField(fields.Raw): "name": value.name, "value": encrypter.obfuscated_token(value.value), "value_type": value.value_type.value, + "description": value.description, } if isinstance(value, Variable): return { @@ -24,6 +25,7 @@ class EnvironmentVariableField(fields.Raw): "name": value.name, "value": value.value, "value_type": value.value_type.value, + "description": value.description, } if isinstance(value, dict): value_type = value.get("value_type") diff --git a/api/libs/helper.py b/api/libs/helper.py index 3f2a630956..48126461a3 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -25,6 +25,31 @@ from extensions.ext_redis import redis_client if TYPE_CHECKING: from models.account import Account + from models.model import EndUser + + +def extract_tenant_id(user: Union["Account", "EndUser"]) -> str | None: + """ + Extract tenant_id from Account or EndUser object. + + Args: + user: Account or EndUser object + + Returns: + tenant_id string if available, None otherwise + + Raises: + ValueError: If user is neither Account nor EndUser + """ + from models.account import Account + from models.model import EndUser + + if isinstance(user, Account): + return user.current_tenant_id + elif isinstance(user, EndUser): + return user.tenant_id + else: + raise ValueError(f"Invalid user type: {type(user)}. Expected Account or EndUser.") def run(script): diff --git a/api/libs/passport.py b/api/libs/passport.py index 8df4f529bc..fe8fc33b5f 100644 --- a/api/libs/passport.py +++ b/api/libs/passport.py @@ -14,9 +14,11 @@ class PassportService: def verify(self, token): try: return jwt.decode(token, self.sk, algorithms=["HS256"]) + except jwt.exceptions.ExpiredSignatureError: + raise Unauthorized("Token has expired.") except jwt.exceptions.InvalidSignatureError: raise Unauthorized("Invalid token signature.") except jwt.exceptions.DecodeError: raise Unauthorized("Invalid token.") - except jwt.exceptions.ExpiredSignatureError: - raise Unauthorized("Token has expired.") + except jwt.exceptions.PyJWTError: # Catch-all for other JWT errors + raise Unauthorized("Invalid token.") diff --git a/api/migrations/versions/2025_06_25_0936-58eb7bdb93fe_add_mcp_server_tool_and_app_server.py b/api/migrations/versions/2025_06_25_0936-58eb7bdb93fe_add_mcp_server_tool_and_app_server.py new file mode 100644 index 0000000000..0548bf05ef --- /dev/null +++ b/api/migrations/versions/2025_06_25_0936-58eb7bdb93fe_add_mcp_server_tool_and_app_server.py @@ -0,0 +1,64 @@ +"""add mcp server tool and app server + +Revision ID: 58eb7bdb93fe +Revises: 0ab65e1cc7fa +Create Date: 2025-06-25 09:36:07.510570 + +""" +from alembic import op +import models as models +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '58eb7bdb93fe' +down_revision = '0ab65e1cc7fa' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('app_mcp_servers', + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', models.types.StringUUID(), nullable=False), + sa.Column('app_id', models.types.StringUUID(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('description', sa.String(length=255), nullable=False), + sa.Column('server_code', sa.String(length=255), nullable=False), + sa.Column('status', sa.String(length=255), server_default=sa.text("'normal'::character varying"), nullable=False), + sa.Column('parameters', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id', name='app_mcp_server_pkey'), + sa.UniqueConstraint('tenant_id', 'app_id', name='unique_app_mcp_server_tenant_app_id'), + sa.UniqueConstraint('server_code', name='unique_app_mcp_server_server_code') + ) + op.create_table('tool_mcp_providers', + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('name', sa.String(length=40), nullable=False), + sa.Column('server_identifier', sa.String(length=24), nullable=False), + sa.Column('server_url', sa.Text(), nullable=False), + sa.Column('server_url_hash', sa.String(length=64), nullable=False), + sa.Column('icon', sa.String(length=255), nullable=True), + sa.Column('tenant_id', models.types.StringUUID(), nullable=False), + sa.Column('user_id', models.types.StringUUID(), nullable=False), + sa.Column('encrypted_credentials', sa.Text(), nullable=True), + sa.Column('authed', sa.Boolean(), nullable=False), + sa.Column('tools', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tool_mcp_provider_pkey'), + sa.UniqueConstraint('tenant_id', 'name', name='unique_mcp_provider_name'), + sa.UniqueConstraint('tenant_id', 'server_identifier', name='unique_mcp_provider_server_identifier'), + sa.UniqueConstraint('tenant_id', 'server_url_hash', name='unique_mcp_provider_server_url') + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tool_mcp_providers') + op.drop_table('app_mcp_servers') + # ### end Alembic commands ### diff --git a/api/models/__init__.py b/api/models/__init__.py index 83b50eb099..1b4bdd32e4 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -34,6 +34,7 @@ from .model import ( App, AppAnnotationHitHistory, AppAnnotationSetting, + AppMCPServer, AppMode, AppModelConfig, Conversation, @@ -103,6 +104,7 @@ __all__ = [ "AppAnnotationHitHistory", "AppAnnotationSetting", "AppDatasetJoin", + "AppMCPServer", # Added "AppMode", "AppModelConfig", "BuiltinToolProvider", diff --git a/api/models/model.py b/api/models/model.py index 93737043d5..7e9e91727d 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -50,7 +50,6 @@ class AppMode(StrEnum): CHAT = "chat" ADVANCED_CHAT = "advanced-chat" AGENT_CHAT = "agent-chat" - CHANNEL = "channel" @classmethod def value_of(cls, value: str) -> "AppMode": @@ -934,7 +933,7 @@ class Message(Base): created_at: Mapped[datetime] = mapped_column(db.DateTime, server_default=func.current_timestamp()) updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp()) agent_based = db.Column(db.Boolean, nullable=False, server_default=db.text("false")) - workflow_run_id = db.Column(StringUUID) + workflow_run_id: Mapped[str] = db.Column(StringUUID) @property def inputs(self): @@ -1456,6 +1455,39 @@ class EndUser(Base, UserMixin): updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp()) +class AppMCPServer(Base): + __tablename__ = "app_mcp_servers" + __table_args__ = ( + db.PrimaryKeyConstraint("id", name="app_mcp_server_pkey"), + db.UniqueConstraint("tenant_id", "app_id", name="unique_app_mcp_server_tenant_app_id"), + db.UniqueConstraint("server_code", name="unique_app_mcp_server_server_code"), + ) + id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) + tenant_id = db.Column(StringUUID, nullable=False) + app_id = db.Column(StringUUID, nullable=False) + name = db.Column(db.String(255), nullable=False) + description = db.Column(db.String(255), nullable=False) + server_code = db.Column(db.String(255), nullable=False) + status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying")) + parameters = db.Column(db.Text, nullable=False) + + created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp()) + updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp()) + + @staticmethod + def generate_server_code(n): + while True: + result = generate_string(n) + while db.session.query(AppMCPServer).filter(AppMCPServer.server_code == result).count() > 0: + result = generate_string(n) + + return result + + @property + def parameters_dict(self) -> dict[str, Any]: + return cast(dict[str, Any], json.loads(self.parameters)) + + class Site(Base): __tablename__ = "sites" __table_args__ = ( diff --git a/api/models/tools.py b/api/models/tools.py index 03fbc3acb1..9d2c3baea5 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -1,12 +1,16 @@ import json from datetime import datetime from typing import Any, cast +from urllib.parse import urlparse import sqlalchemy as sa from deprecated import deprecated from sqlalchemy import ForeignKey, func from sqlalchemy.orm import Mapped, mapped_column +from core.file import helpers as file_helpers +from core.helper import encrypter +from core.mcp.types import Tool from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiToolBundle from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration @@ -189,6 +193,108 @@ class WorkflowToolProvider(Base): return db.session.query(App).filter(App.id == self.app_id).first() +class MCPToolProvider(Base): + """ + The table stores the mcp providers. + """ + + __tablename__ = "tool_mcp_providers" + __table_args__ = ( + db.PrimaryKeyConstraint("id", name="tool_mcp_provider_pkey"), + db.UniqueConstraint("tenant_id", "server_url_hash", name="unique_mcp_provider_server_url"), + db.UniqueConstraint("tenant_id", "name", name="unique_mcp_provider_name"), + db.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"), + ) + + id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()")) + # name of the mcp provider + name: Mapped[str] = mapped_column(db.String(40), nullable=False) + # server identifier of the mcp provider + server_identifier: Mapped[str] = mapped_column(db.String(24), nullable=False) + # encrypted url of the mcp provider + server_url: Mapped[str] = mapped_column(db.Text, nullable=False) + # hash of server_url for uniqueness check + server_url_hash: Mapped[str] = mapped_column(db.String(64), nullable=False) + # icon of the mcp provider + icon: Mapped[str] = mapped_column(db.String(255), nullable=True) + # tenant id + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + # who created this tool + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + # encrypted credentials + encrypted_credentials: Mapped[str] = mapped_column(db.Text, nullable=True) + # authed + authed: Mapped[bool] = mapped_column(db.Boolean, nullable=False, default=False) + # tools + tools: Mapped[str] = mapped_column(db.Text, nullable=False, default="[]") + created_at: Mapped[datetime] = mapped_column( + db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)") + ) + updated_at: Mapped[datetime] = mapped_column( + db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)") + ) + + def load_user(self) -> Account | None: + return db.session.query(Account).filter(Account.id == self.user_id).first() + + @property + def tenant(self) -> Tenant | None: + return db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first() + + @property + def credentials(self) -> dict: + try: + return cast(dict, json.loads(self.encrypted_credentials)) or {} + except Exception: + return {} + + @property + def mcp_tools(self) -> list[Tool]: + return [Tool(**tool) for tool in json.loads(self.tools)] + + @property + def provider_icon(self) -> dict[str, str] | str: + try: + return cast(dict[str, str], json.loads(self.icon)) + except json.JSONDecodeError: + return file_helpers.get_signed_file_url(self.icon) + + @property + def decrypted_server_url(self) -> str: + return cast(str, encrypter.decrypt_token(self.tenant_id, self.server_url)) + + @property + def masked_server_url(self) -> str: + def mask_url(url: str, mask_char: str = "*") -> str: + """ + mask the url to a simple string + """ + parsed = urlparse(url) + base_url = f"{parsed.scheme}://{parsed.netloc}" + + if parsed.path and parsed.path != "/": + return f"{base_url}/{mask_char * 6}" + else: + return base_url + + return mask_url(self.decrypted_server_url) + + @property + def decrypted_credentials(self) -> dict: + from core.tools.mcp_tool.provider import MCPToolProviderController + from core.tools.utils.configuration import ProviderConfigEncrypter + + provider_controller = MCPToolProviderController._from_db(self) + + tool_configuration = ProviderConfigEncrypter( + tenant_id=self.tenant_id, + config=list(provider_controller.get_credentials_schema()), + provider_type=provider_controller.provider_type.value, + provider_identity=provider_controller.provider_id, + ) + return tool_configuration.decrypt(self.credentials, use_cache=False) + + class ToolModelInvoke(Base): """ store the invoke logs from tool invoke diff --git a/api/models/workflow.py b/api/models/workflow.py index 7f01135af3..77d48bec4f 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -15,6 +15,7 @@ from core.variables import utils as variable_utils from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID from core.workflow.nodes.enums import NodeType from factories.variable_factory import TypeMismatchError, build_segment_with_type +from libs.helper import extract_tenant_id from ._workflow_exc import NodeNotFoundError, WorkflowDataError @@ -352,12 +353,7 @@ class Workflow(Base): self._environment_variables = "{}" # Get tenant_id from current_user (Account or EndUser) - if isinstance(current_user, Account): - # Account user - tenant_id = current_user.current_tenant_id - else: - # EndUser - tenant_id = current_user.tenant_id + tenant_id = extract_tenant_id(current_user) if not tenant_id: return [] @@ -384,12 +380,7 @@ class Workflow(Base): return # Get tenant_id from current_user (Account or EndUser) - if isinstance(current_user, Account): - # Account user - tenant_id = current_user.current_tenant_id - else: - # EndUser - tenant_id = current_user.tenant_id + tenant_id = extract_tenant_id(current_user) if not tenant_id: self._environment_variables = "{}" diff --git a/api/pyproject.toml b/api/pyproject.toml index d33806d0ae..7f1efa671f 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dify-api" -version = "1.5.1" +version = "1.6.0" requires-python = ">=3.11,<3.13" dependencies = [ @@ -82,6 +82,8 @@ dependencies = [ "weave~=0.51.0", "yarl~=1.18.3", "webvtt-py~=0.5.1", + "sseclient-py>=1.8.0", + "httpx-sse>=0.4.0", "sendgrid~=6.12.3", ] # Before adding new dependency, consider place it in @@ -106,7 +108,7 @@ dev = [ "faker~=32.1.0", "lxml-stubs~=0.5.1", "mypy~=1.16.0", - "ruff~=0.11.5", + "ruff~=0.12.3", "pytest~=8.3.2", "pytest-benchmark~=4.0.0", "pytest-cov~=4.1.0", diff --git a/api/repositories/__init__.py b/api/repositories/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/repositories/api_workflow_node_execution_repository.py b/api/repositories/api_workflow_node_execution_repository.py new file mode 100644 index 0000000000..00a2d1f87d --- /dev/null +++ b/api/repositories/api_workflow_node_execution_repository.py @@ -0,0 +1,197 @@ +""" +Service-layer repository protocol for WorkflowNodeExecutionModel operations. + +This module provides a protocol interface for service-layer operations on WorkflowNodeExecutionModel +that abstracts database queries currently done directly in service classes. This repository is +specifically designed for service-layer needs and is separate from the core domain repository. + +The service repository handles operations that require access to database-specific fields like +tenant_id, app_id, triggered_from, etc., which are not part of the core domain model. +""" + +from collections.abc import Sequence +from datetime import datetime +from typing import Optional, Protocol + +from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository +from models.workflow import WorkflowNodeExecutionModel + + +class DifyAPIWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository, Protocol): + """ + Protocol for service-layer operations on WorkflowNodeExecutionModel. + + This repository provides database access patterns specifically needed by service classes, + handling queries that involve database-specific fields and multi-tenancy concerns. + + Key responsibilities: + - Manages database operations for workflow node executions + - Handles multi-tenant data isolation + - Provides batch processing capabilities + - Supports execution lifecycle management + + Implementation notes: + - Returns database models directly (WorkflowNodeExecutionModel) + - Handles tenant/app filtering automatically + - Provides service-specific query patterns + - Focuses on database operations without domain logic + - Supports cleanup and maintenance operations + """ + + def get_node_last_execution( + self, + tenant_id: str, + app_id: str, + workflow_id: str, + node_id: str, + ) -> Optional[WorkflowNodeExecutionModel]: + """ + Get the most recent execution for a specific node. + + This method finds the latest execution of a specific node within a workflow, + ordered by creation time. Used primarily for debugging and inspection purposes. + + Args: + tenant_id: The tenant identifier + app_id: The application identifier + workflow_id: The workflow identifier + node_id: The node identifier + + Returns: + The most recent WorkflowNodeExecutionModel for the node, or None if not found + """ + ... + + def get_executions_by_workflow_run( + self, + tenant_id: str, + app_id: str, + workflow_run_id: str, + ) -> Sequence[WorkflowNodeExecutionModel]: + """ + Get all node executions for a specific workflow run. + + This method retrieves all node executions that belong to a specific workflow run, + ordered by index in descending order for proper trace visualization. + + Args: + tenant_id: The tenant identifier + app_id: The application identifier + workflow_run_id: The workflow run identifier + + Returns: + A sequence of WorkflowNodeExecutionModel instances ordered by index (desc) + """ + ... + + def get_execution_by_id( + self, + execution_id: str, + tenant_id: Optional[str] = None, + ) -> Optional[WorkflowNodeExecutionModel]: + """ + Get a workflow node execution by its ID. + + This method retrieves a specific execution by its unique identifier. + Tenant filtering is optional for cases where the execution ID is globally unique. + + When `tenant_id` is None, it's the caller's responsibility to ensure proper data isolation between tenants. + If the `execution_id` comes from untrusted sources (e.g., retrieved from an API request), the caller should + set `tenant_id` to prevent horizontal privilege escalation. + + Args: + execution_id: The execution identifier + tenant_id: Optional tenant identifier for additional filtering + + Returns: + The WorkflowNodeExecutionModel if found, or None if not found + """ + ... + + def delete_expired_executions( + self, + tenant_id: str, + before_date: datetime, + batch_size: int = 1000, + ) -> int: + """ + Delete workflow node executions that are older than the specified date. + + This method is used for cleanup operations to remove expired executions + in batches to avoid overwhelming the database. + + Args: + tenant_id: The tenant identifier + before_date: Delete executions created before this date + batch_size: Maximum number of executions to delete in one batch + + Returns: + The number of executions deleted + """ + ... + + def delete_executions_by_app( + self, + tenant_id: str, + app_id: str, + batch_size: int = 1000, + ) -> int: + """ + Delete all workflow node executions for a specific app. + + This method is used when removing an app and all its related data. + Executions are deleted in batches to avoid overwhelming the database. + + Args: + tenant_id: The tenant identifier + app_id: The application identifier + batch_size: Maximum number of executions to delete in one batch + + Returns: + The total number of executions deleted + """ + ... + + def get_expired_executions_batch( + self, + tenant_id: str, + before_date: datetime, + batch_size: int = 1000, + ) -> Sequence[WorkflowNodeExecutionModel]: + """ + Get a batch of expired workflow node executions for backup purposes. + + This method retrieves expired executions without deleting them, + allowing the caller to backup the data before deletion. + + Args: + tenant_id: The tenant identifier + before_date: Get executions created before this date + batch_size: Maximum number of executions to retrieve + + Returns: + A sequence of WorkflowNodeExecutionModel instances + """ + ... + + def delete_executions_by_ids( + self, + execution_ids: Sequence[str], + ) -> int: + """ + Delete workflow node executions by their IDs. + + This method deletes specific executions by their IDs, + typically used after backing up the data. + + This method does not perform tenant isolation checks. The caller is responsible for ensuring proper + data isolation between tenants. When execution IDs come from untrusted sources (e.g., API requests), + additional tenant validation should be implemented to prevent unauthorized access. + + Args: + execution_ids: List of execution IDs to delete + + Returns: + The number of executions deleted + """ + ... diff --git a/api/repositories/api_workflow_run_repository.py b/api/repositories/api_workflow_run_repository.py new file mode 100644 index 0000000000..59e7baeb79 --- /dev/null +++ b/api/repositories/api_workflow_run_repository.py @@ -0,0 +1,181 @@ +""" +API WorkflowRun Repository Protocol + +This module defines the protocol for service-layer WorkflowRun operations. +The repository provides an abstraction layer for WorkflowRun database operations +used by service classes, separating service-layer concerns from core domain logic. + +Key Features: +- Paginated workflow run queries with filtering +- Bulk deletion operations with OSS backup support +- Multi-tenant data isolation +- Expired record cleanup with data retention +- Service-layer specific query patterns + +Usage: + This protocol should be used by service classes that need to perform + WorkflowRun database operations. It provides a clean interface that + hides implementation details and supports dependency injection. + +Example: + ```python + from repositories.dify_api_repository_factory import DifyAPIRepositoryFactory + + session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker) + + # Get paginated workflow runs + runs = repo.get_paginated_workflow_runs( + tenant_id="tenant-123", + app_id="app-456", + triggered_from="debugging", + limit=20 + ) + ``` +""" + +from collections.abc import Sequence +from datetime import datetime +from typing import Optional, Protocol + +from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository +from libs.infinite_scroll_pagination import InfiniteScrollPagination +from models.workflow import WorkflowRun + + +class APIWorkflowRunRepository(WorkflowExecutionRepository, Protocol): + """ + Protocol for service-layer WorkflowRun repository operations. + + This protocol defines the interface for WorkflowRun database operations + that are specific to service-layer needs, including pagination, filtering, + and bulk operations with data backup support. + """ + + def get_paginated_workflow_runs( + self, + tenant_id: str, + app_id: str, + triggered_from: str, + limit: int = 20, + last_id: Optional[str] = None, + ) -> InfiniteScrollPagination: + """ + Get paginated workflow runs with filtering. + + Retrieves workflow runs for a specific app and trigger source with + cursor-based pagination support. Used primarily for debugging and + workflow run listing in the UI. + + Args: + tenant_id: Tenant identifier for multi-tenant isolation + app_id: Application identifier + triggered_from: Filter by trigger source (e.g., "debugging", "app-run") + limit: Maximum number of records to return (default: 20) + last_id: Cursor for pagination - ID of the last record from previous page + + Returns: + InfiniteScrollPagination object containing: + - data: List of WorkflowRun objects + - limit: Applied limit + - has_more: Boolean indicating if more records exist + + Raises: + ValueError: If last_id is provided but the corresponding record doesn't exist + """ + ... + + def get_workflow_run_by_id( + self, + tenant_id: str, + app_id: str, + run_id: str, + ) -> Optional[WorkflowRun]: + """ + Get a specific workflow run by ID. + + Retrieves a single workflow run with tenant and app isolation. + Used for workflow run detail views and execution tracking. + + Args: + tenant_id: Tenant identifier for multi-tenant isolation + app_id: Application identifier + run_id: Workflow run identifier + + Returns: + WorkflowRun object if found, None otherwise + """ + ... + + def get_expired_runs_batch( + self, + tenant_id: str, + before_date: datetime, + batch_size: int = 1000, + ) -> Sequence[WorkflowRun]: + """ + Get a batch of expired workflow runs for cleanup. + + Retrieves workflow runs created before the specified date for + cleanup operations. Used by scheduled tasks to remove old data + while maintaining data retention policies. + + Args: + tenant_id: Tenant identifier for multi-tenant isolation + before_date: Only return runs created before this date + batch_size: Maximum number of records to return + + Returns: + Sequence of WorkflowRun objects to be processed for cleanup + """ + ... + + def delete_runs_by_ids( + self, + run_ids: Sequence[str], + ) -> int: + """ + Delete workflow runs by their IDs. + + Performs bulk deletion of workflow runs by ID. This method should + be used after backing up the data to OSS storage for retention. + + Args: + run_ids: Sequence of workflow run IDs to delete + + Returns: + Number of records actually deleted + + Note: + This method performs hard deletion. Ensure data is backed up + to OSS storage before calling this method for compliance with + data retention policies. + """ + ... + + def delete_runs_by_app( + self, + tenant_id: str, + app_id: str, + batch_size: int = 1000, + ) -> int: + """ + Delete all workflow runs for a specific app. + + Performs bulk deletion of all workflow runs associated with an app. + Used during app cleanup operations. Processes records in batches + to avoid memory issues and long-running transactions. + + Args: + tenant_id: Tenant identifier for multi-tenant isolation + app_id: Application identifier + batch_size: Number of records to process in each batch + + Returns: + Total number of records deleted across all batches + + Note: + This method performs hard deletion without backup. Use with caution + and ensure proper data retention policies are followed. + """ + ... diff --git a/api/repositories/factory.py b/api/repositories/factory.py new file mode 100644 index 0000000000..0a0adbf2c2 --- /dev/null +++ b/api/repositories/factory.py @@ -0,0 +1,103 @@ +""" +DifyAPI Repository Factory for creating repository instances. + +This factory is specifically designed for DifyAPI repositories that handle +service-layer operations with dependency injection patterns. +""" + +import logging + +from sqlalchemy.orm import sessionmaker + +from configs import dify_config +from core.repositories import DifyCoreRepositoryFactory, RepositoryImportError +from repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository +from repositories.api_workflow_run_repository import APIWorkflowRunRepository + +logger = logging.getLogger(__name__) + + +class DifyAPIRepositoryFactory(DifyCoreRepositoryFactory): + """ + Factory for creating DifyAPI repository instances based on configuration. + + This factory handles the creation of repositories that are specifically designed + for service-layer operations and use dependency injection with sessionmaker + for better testability and separation of concerns. + """ + + @classmethod + def create_api_workflow_node_execution_repository( + cls, session_maker: sessionmaker + ) -> DifyAPIWorkflowNodeExecutionRepository: + """ + Create a DifyAPIWorkflowNodeExecutionRepository instance based on configuration. + + This repository is designed for service-layer operations and uses dependency injection + with a sessionmaker for better testability and separation of concerns. It provides + database access patterns specifically needed by service classes, handling queries + that involve database-specific fields and multi-tenancy concerns. + + Args: + session_maker: SQLAlchemy sessionmaker to inject for database session management. + + Returns: + Configured DifyAPIWorkflowNodeExecutionRepository instance + + Raises: + RepositoryImportError: If the configured repository cannot be imported or instantiated + """ + class_path = dify_config.API_WORKFLOW_NODE_EXECUTION_REPOSITORY + logger.debug(f"Creating DifyAPIWorkflowNodeExecutionRepository from: {class_path}") + + try: + repository_class = cls._import_class(class_path) + cls._validate_repository_interface(repository_class, DifyAPIWorkflowNodeExecutionRepository) + # Service repository requires session_maker parameter + cls._validate_constructor_signature(repository_class, ["session_maker"]) + + return repository_class(session_maker=session_maker) # type: ignore[no-any-return] + except RepositoryImportError: + # Re-raise our custom errors as-is + raise + except Exception as e: + logger.exception("Failed to create DifyAPIWorkflowNodeExecutionRepository") + raise RepositoryImportError( + f"Failed to create DifyAPIWorkflowNodeExecutionRepository from '{class_path}': {e}" + ) from e + + @classmethod + def create_api_workflow_run_repository(cls, session_maker: sessionmaker) -> APIWorkflowRunRepository: + """ + Create an APIWorkflowRunRepository instance based on configuration. + + This repository is designed for service-layer WorkflowRun operations and uses dependency + injection with a sessionmaker for better testability and separation of concerns. It provides + database access patterns specifically needed by service classes for workflow run management, + including pagination, filtering, and bulk operations. + + Args: + session_maker: SQLAlchemy sessionmaker to inject for database session management. + + Returns: + Configured APIWorkflowRunRepository instance + + Raises: + RepositoryImportError: If the configured repository cannot be imported or instantiated + """ + class_path = dify_config.API_WORKFLOW_RUN_REPOSITORY + logger.debug(f"Creating APIWorkflowRunRepository from: {class_path}") + + try: + repository_class = cls._import_class(class_path) + cls._validate_repository_interface(repository_class, APIWorkflowRunRepository) + # Service repository requires session_maker parameter + cls._validate_constructor_signature(repository_class, ["session_maker"]) + + return repository_class(session_maker=session_maker) # type: ignore[no-any-return] + except RepositoryImportError: + # Re-raise our custom errors as-is + raise + except Exception as e: + logger.exception("Failed to create APIWorkflowRunRepository") + raise RepositoryImportError(f"Failed to create APIWorkflowRunRepository from '{class_path}': {e}") from e diff --git a/api/repositories/sqlalchemy_api_workflow_node_execution_repository.py b/api/repositories/sqlalchemy_api_workflow_node_execution_repository.py new file mode 100644 index 0000000000..e6a23ddf9f --- /dev/null +++ b/api/repositories/sqlalchemy_api_workflow_node_execution_repository.py @@ -0,0 +1,290 @@ +""" +SQLAlchemy implementation of WorkflowNodeExecutionServiceRepository. + +This module provides a concrete implementation of the service repository protocol +using SQLAlchemy 2.0 style queries for WorkflowNodeExecutionModel operations. +""" + +from collections.abc import Sequence +from datetime import datetime +from typing import Optional + +from sqlalchemy import delete, desc, select +from sqlalchemy.orm import Session, sessionmaker + +from models.workflow import WorkflowNodeExecutionModel +from repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository + + +class DifyAPISQLAlchemyWorkflowNodeExecutionRepository(DifyAPIWorkflowNodeExecutionRepository): + """ + SQLAlchemy implementation of DifyAPIWorkflowNodeExecutionRepository. + + This repository provides service-layer database operations for WorkflowNodeExecutionModel + using SQLAlchemy 2.0 style queries. It implements the DifyAPIWorkflowNodeExecutionRepository + protocol with the following features: + + - Multi-tenancy data isolation through tenant_id filtering + - Direct database model operations without domain conversion + - Batch processing for efficient large-scale operations + - Optimized query patterns for common access patterns + - Dependency injection for better testability and maintainability + - Session management and transaction handling with proper cleanup + - Maintenance operations for data lifecycle management + - Thread-safe database operations using session-per-request pattern + """ + + def __init__(self, session_maker: sessionmaker[Session]): + """ + Initialize the repository with a sessionmaker. + + Args: + session_maker: SQLAlchemy sessionmaker for creating database sessions + """ + self._session_maker = session_maker + + def get_node_last_execution( + self, + tenant_id: str, + app_id: str, + workflow_id: str, + node_id: str, + ) -> Optional[WorkflowNodeExecutionModel]: + """ + Get the most recent execution for a specific node. + + This method replicates the query pattern from WorkflowService.get_node_last_run() + using SQLAlchemy 2.0 style syntax. + + Args: + tenant_id: The tenant identifier + app_id: The application identifier + workflow_id: The workflow identifier + node_id: The node identifier + + Returns: + The most recent WorkflowNodeExecutionModel for the node, or None if not found + """ + stmt = ( + select(WorkflowNodeExecutionModel) + .where( + WorkflowNodeExecutionModel.tenant_id == tenant_id, + WorkflowNodeExecutionModel.app_id == app_id, + WorkflowNodeExecutionModel.workflow_id == workflow_id, + WorkflowNodeExecutionModel.node_id == node_id, + ) + .order_by(desc(WorkflowNodeExecutionModel.created_at)) + .limit(1) + ) + + with self._session_maker() as session: + return session.scalar(stmt) + + def get_executions_by_workflow_run( + self, + tenant_id: str, + app_id: str, + workflow_run_id: str, + ) -> Sequence[WorkflowNodeExecutionModel]: + """ + Get all node executions for a specific workflow run. + + This method replicates the query pattern from WorkflowRunService.get_workflow_run_node_executions() + using SQLAlchemy 2.0 style syntax. + + Args: + tenant_id: The tenant identifier + app_id: The application identifier + workflow_run_id: The workflow run identifier + + Returns: + A sequence of WorkflowNodeExecutionModel instances ordered by index (desc) + """ + stmt = ( + select(WorkflowNodeExecutionModel) + .where( + WorkflowNodeExecutionModel.tenant_id == tenant_id, + WorkflowNodeExecutionModel.app_id == app_id, + WorkflowNodeExecutionModel.workflow_run_id == workflow_run_id, + ) + .order_by(desc(WorkflowNodeExecutionModel.index)) + ) + + with self._session_maker() as session: + return session.execute(stmt).scalars().all() + + def get_execution_by_id( + self, + execution_id: str, + tenant_id: Optional[str] = None, + ) -> Optional[WorkflowNodeExecutionModel]: + """ + Get a workflow node execution by its ID. + + This method replicates the query pattern from WorkflowDraftVariableService + and WorkflowService.single_step_run_workflow_node() using SQLAlchemy 2.0 style syntax. + + When `tenant_id` is None, it's the caller's responsibility to ensure proper data isolation between tenants. + If the `execution_id` comes from untrusted sources (e.g., retrieved from an API request), the caller should + set `tenant_id` to prevent horizontal privilege escalation. + + Args: + execution_id: The execution identifier + tenant_id: Optional tenant identifier for additional filtering + + Returns: + The WorkflowNodeExecutionModel if found, or None if not found + """ + stmt = select(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.id == execution_id) + + # Add tenant filtering if provided + if tenant_id is not None: + stmt = stmt.where(WorkflowNodeExecutionModel.tenant_id == tenant_id) + + with self._session_maker() as session: + return session.scalar(stmt) + + def delete_expired_executions( + self, + tenant_id: str, + before_date: datetime, + batch_size: int = 1000, + ) -> int: + """ + Delete workflow node executions that are older than the specified date. + + Args: + tenant_id: The tenant identifier + before_date: Delete executions created before this date + batch_size: Maximum number of executions to delete in one batch + + Returns: + The number of executions deleted + """ + total_deleted = 0 + + while True: + with self._session_maker() as session: + # Find executions to delete in batches + stmt = ( + select(WorkflowNodeExecutionModel.id) + .where( + WorkflowNodeExecutionModel.tenant_id == tenant_id, + WorkflowNodeExecutionModel.created_at < before_date, + ) + .limit(batch_size) + ) + + execution_ids = session.execute(stmt).scalars().all() + if not execution_ids: + break + + # Delete the batch + delete_stmt = delete(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.id.in_(execution_ids)) + result = session.execute(delete_stmt) + session.commit() + total_deleted += result.rowcount + + # If we deleted fewer than the batch size, we're done + if len(execution_ids) < batch_size: + break + + return total_deleted + + def delete_executions_by_app( + self, + tenant_id: str, + app_id: str, + batch_size: int = 1000, + ) -> int: + """ + Delete all workflow node executions for a specific app. + + Args: + tenant_id: The tenant identifier + app_id: The application identifier + batch_size: Maximum number of executions to delete in one batch + + Returns: + The total number of executions deleted + """ + total_deleted = 0 + + while True: + with self._session_maker() as session: + # Find executions to delete in batches + stmt = ( + select(WorkflowNodeExecutionModel.id) + .where( + WorkflowNodeExecutionModel.tenant_id == tenant_id, + WorkflowNodeExecutionModel.app_id == app_id, + ) + .limit(batch_size) + ) + + execution_ids = session.execute(stmt).scalars().all() + if not execution_ids: + break + + # Delete the batch + delete_stmt = delete(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.id.in_(execution_ids)) + result = session.execute(delete_stmt) + session.commit() + total_deleted += result.rowcount + + # If we deleted fewer than the batch size, we're done + if len(execution_ids) < batch_size: + break + + return total_deleted + + def get_expired_executions_batch( + self, + tenant_id: str, + before_date: datetime, + batch_size: int = 1000, + ) -> Sequence[WorkflowNodeExecutionModel]: + """ + Get a batch of expired workflow node executions for backup purposes. + + Args: + tenant_id: The tenant identifier + before_date: Get executions created before this date + batch_size: Maximum number of executions to retrieve + + Returns: + A sequence of WorkflowNodeExecutionModel instances + """ + stmt = ( + select(WorkflowNodeExecutionModel) + .where( + WorkflowNodeExecutionModel.tenant_id == tenant_id, + WorkflowNodeExecutionModel.created_at < before_date, + ) + .limit(batch_size) + ) + + with self._session_maker() as session: + return session.execute(stmt).scalars().all() + + def delete_executions_by_ids( + self, + execution_ids: Sequence[str], + ) -> int: + """ + Delete workflow node executions by their IDs. + + Args: + execution_ids: List of execution IDs to delete + + Returns: + The number of executions deleted + """ + if not execution_ids: + return 0 + + with self._session_maker() as session: + stmt = delete(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.id.in_(execution_ids)) + result = session.execute(stmt) + session.commit() + return result.rowcount diff --git a/api/repositories/sqlalchemy_api_workflow_run_repository.py b/api/repositories/sqlalchemy_api_workflow_run_repository.py new file mode 100644 index 0000000000..ebd1d74b20 --- /dev/null +++ b/api/repositories/sqlalchemy_api_workflow_run_repository.py @@ -0,0 +1,203 @@ +""" +SQLAlchemy API WorkflowRun Repository Implementation + +This module provides the SQLAlchemy-based implementation of the APIWorkflowRunRepository +protocol. It handles service-layer WorkflowRun database operations using SQLAlchemy 2.0 +style queries with proper session management and multi-tenant data isolation. + +Key Features: +- SQLAlchemy 2.0 style queries for modern database operations +- Cursor-based pagination for efficient large dataset handling +- Bulk operations with batch processing for performance +- Multi-tenant data isolation and security +- Proper session management with dependency injection + +Implementation Notes: +- Uses sessionmaker for consistent session management +- Implements cursor-based pagination using created_at timestamps +- Provides efficient bulk deletion with batch processing +- Maintains data consistency with proper transaction handling +""" + +import logging +from collections.abc import Sequence +from datetime import datetime +from typing import Optional, cast + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session, sessionmaker + +from libs.infinite_scroll_pagination import InfiniteScrollPagination +from models.workflow import WorkflowRun +from repositories.api_workflow_run_repository import APIWorkflowRunRepository + +logger = logging.getLogger(__name__) + + +class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): + """ + SQLAlchemy implementation of APIWorkflowRunRepository. + + Provides service-layer WorkflowRun database operations using SQLAlchemy 2.0 + style queries. Supports dependency injection through sessionmaker and + maintains proper multi-tenant data isolation. + + Args: + session_maker: SQLAlchemy sessionmaker instance for database connections + """ + + def __init__(self, session_maker: sessionmaker[Session]) -> None: + """ + Initialize the repository with a sessionmaker. + + Args: + session_maker: SQLAlchemy sessionmaker for database connections + """ + self._session_maker = session_maker + + def get_paginated_workflow_runs( + self, + tenant_id: str, + app_id: str, + triggered_from: str, + limit: int = 20, + last_id: Optional[str] = None, + ) -> InfiniteScrollPagination: + """ + Get paginated workflow runs with filtering. + + Implements cursor-based pagination using created_at timestamps for + efficient handling of large datasets. Filters by tenant, app, and + trigger source for proper data isolation. + """ + with self._session_maker() as session: + # Build base query with filters + base_stmt = select(WorkflowRun).where( + WorkflowRun.tenant_id == tenant_id, + WorkflowRun.app_id == app_id, + WorkflowRun.triggered_from == triggered_from, + ) + + if last_id: + # Get the last workflow run for cursor-based pagination + last_run_stmt = base_stmt.where(WorkflowRun.id == last_id) + last_workflow_run = session.scalar(last_run_stmt) + + if not last_workflow_run: + raise ValueError("Last workflow run not exists") + + # Get records created before the last run's timestamp + base_stmt = base_stmt.where( + WorkflowRun.created_at < last_workflow_run.created_at, + WorkflowRun.id != last_workflow_run.id, + ) + + # First page - get most recent records + workflow_runs = session.scalars(base_stmt.order_by(WorkflowRun.created_at.desc()).limit(limit + 1)).all() + + # Check if there are more records for pagination + has_more = len(workflow_runs) > limit + if has_more: + workflow_runs = workflow_runs[:-1] + + return InfiniteScrollPagination(data=workflow_runs, limit=limit, has_more=has_more) + + def get_workflow_run_by_id( + self, + tenant_id: str, + app_id: str, + run_id: str, + ) -> Optional[WorkflowRun]: + """ + Get a specific workflow run by ID with tenant and app isolation. + """ + with self._session_maker() as session: + stmt = select(WorkflowRun).where( + WorkflowRun.tenant_id == tenant_id, + WorkflowRun.app_id == app_id, + WorkflowRun.id == run_id, + ) + return cast(Optional[WorkflowRun], session.scalar(stmt)) + + def get_expired_runs_batch( + self, + tenant_id: str, + before_date: datetime, + batch_size: int = 1000, + ) -> Sequence[WorkflowRun]: + """ + Get a batch of expired workflow runs for cleanup operations. + """ + with self._session_maker() as session: + stmt = ( + select(WorkflowRun) + .where( + WorkflowRun.tenant_id == tenant_id, + WorkflowRun.created_at < before_date, + ) + .limit(batch_size) + ) + return cast(Sequence[WorkflowRun], session.scalars(stmt).all()) + + def delete_runs_by_ids( + self, + run_ids: Sequence[str], + ) -> int: + """ + Delete workflow runs by their IDs using bulk deletion. + """ + if not run_ids: + return 0 + + with self._session_maker() as session: + stmt = delete(WorkflowRun).where(WorkflowRun.id.in_(run_ids)) + result = session.execute(stmt) + session.commit() + + deleted_count = cast(int, result.rowcount) + logger.info(f"Deleted {deleted_count} workflow runs by IDs") + return deleted_count + + def delete_runs_by_app( + self, + tenant_id: str, + app_id: str, + batch_size: int = 1000, + ) -> int: + """ + Delete all workflow runs for a specific app in batches. + """ + total_deleted = 0 + + while True: + with self._session_maker() as session: + # Get a batch of run IDs to delete + stmt = ( + select(WorkflowRun.id) + .where( + WorkflowRun.tenant_id == tenant_id, + WorkflowRun.app_id == app_id, + ) + .limit(batch_size) + ) + run_ids = session.scalars(stmt).all() + + if not run_ids: + break + + # Delete the batch + delete_stmt = delete(WorkflowRun).where(WorkflowRun.id.in_(run_ids)) + result = session.execute(delete_stmt) + session.commit() + + batch_deleted = result.rowcount + total_deleted += batch_deleted + + logger.info(f"Deleted batch of {batch_deleted} workflow runs for app {app_id}") + + # If we deleted fewer records than the batch size, we're done + if batch_deleted < batch_size: + break + + logger.info(f"Total deleted {total_deleted} workflow runs for app {app_id}") + return total_deleted diff --git a/api/services/account_service.py b/api/services/account_service.py index 3fdbda48a6..2ba6f4345b 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -16,7 +16,7 @@ from configs import dify_config from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created from extensions.ext_database import db -from extensions.ext_redis import redis_client +from extensions.ext_redis import redis_client, redis_fallback from libs.helper import RateLimiter, TokenManager from libs.passport import PassportService from libs.password import compare_password, hash_password, valid_password @@ -495,6 +495,7 @@ class AccountService: return account @staticmethod + @redis_fallback(default_return=None) def add_login_error_rate_limit(email: str) -> None: key = f"login_error_rate_limit:{email}" count = redis_client.get(key) @@ -504,6 +505,7 @@ class AccountService: redis_client.setex(key, dify_config.LOGIN_LOCKOUT_DURATION, count) @staticmethod + @redis_fallback(default_return=False) def is_login_error_rate_limit(email: str) -> bool: key = f"login_error_rate_limit:{email}" count = redis_client.get(key) @@ -516,11 +518,13 @@ class AccountService: return False @staticmethod + @redis_fallback(default_return=None) def reset_login_error_rate_limit(email: str): key = f"login_error_rate_limit:{email}" redis_client.delete(key) @staticmethod + @redis_fallback(default_return=None) def add_forgot_password_error_rate_limit(email: str) -> None: key = f"forgot_password_error_rate_limit:{email}" count = redis_client.get(key) @@ -530,6 +534,7 @@ class AccountService: redis_client.setex(key, dify_config.FORGOT_PASSWORD_LOCKOUT_DURATION, count) @staticmethod + @redis_fallback(default_return=False) def is_forgot_password_error_rate_limit(email: str) -> bool: key = f"forgot_password_error_rate_limit:{email}" count = redis_client.get(key) @@ -542,11 +547,13 @@ class AccountService: return False @staticmethod + @redis_fallback(default_return=None) def reset_forgot_password_error_rate_limit(email: str): key = f"forgot_password_error_rate_limit:{email}" redis_client.delete(key) @staticmethod + @redis_fallback(default_return=False) def is_email_send_ip_limit(ip_address: str): minute_key = f"email_send_ip_limit_minute:{ip_address}" freeze_key = f"email_send_ip_limit_freeze:{ip_address}" diff --git a/api/services/app_service.py b/api/services/app_service.py index d08462d001..db0f8cd414 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -47,8 +47,6 @@ class AppService: filters.append(App.mode == AppMode.ADVANCED_CHAT.value) elif args["mode"] == "agent-chat": filters.append(App.mode == AppMode.AGENT_CHAT.value) - elif args["mode"] == "channel": - filters.append(App.mode == AppMode.CHANNEL.value) if args.get("is_created_by_me", False): filters.append(App.created_by == user_id) diff --git a/api/services/clear_free_plan_tenant_expired_logs.py b/api/services/clear_free_plan_tenant_expired_logs.py index 1fd560d581..ddd16b2e0c 100644 --- a/api/services/clear_free_plan_tenant_expired_logs.py +++ b/api/services/clear_free_plan_tenant_expired_logs.py @@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import click from flask import Flask, current_app -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from configs import dify_config from core.model_runtime.utils.encoders import jsonable_encoder @@ -14,7 +14,7 @@ from extensions.ext_database import db from extensions.ext_storage import storage from models.account import Tenant from models.model import App, Conversation, Message -from models.workflow import WorkflowNodeExecutionModel, WorkflowRun +from repositories.factory import DifyAPIRepositoryFactory from services.billing_service import BillingService logger = logging.getLogger(__name__) @@ -105,84 +105,99 @@ class ClearFreePlanTenantExpiredLogs: ) ) - while True: - with Session(db.engine).no_autoflush as session: - workflow_node_executions = ( - session.query(WorkflowNodeExecutionModel) - .filter( - WorkflowNodeExecutionModel.tenant_id == tenant_id, - WorkflowNodeExecutionModel.created_at - < datetime.datetime.now() - datetime.timedelta(days=days), - ) - .limit(batch) - .all() - ) + # Process expired workflow node executions with backup + session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + node_execution_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(session_maker) + before_date = datetime.datetime.now() - datetime.timedelta(days=days) + total_deleted = 0 - if len(workflow_node_executions) == 0: - break + while True: + # Get a batch of expired executions for backup + workflow_node_executions = node_execution_repo.get_expired_executions_batch( + tenant_id=tenant_id, + before_date=before_date, + batch_size=batch, + ) - # save workflow node executions - storage.save( - f"free_plan_tenant_expired_logs/" - f"{tenant_id}/workflow_node_executions/{datetime.datetime.now().strftime('%Y-%m-%d')}" - f"-{time.time()}.json", - json.dumps( - jsonable_encoder(workflow_node_executions), - ).encode("utf-8"), - ) + if len(workflow_node_executions) == 0: + break + + # Save workflow node executions to storage + storage.save( + f"free_plan_tenant_expired_logs/" + f"{tenant_id}/workflow_node_executions/{datetime.datetime.now().strftime('%Y-%m-%d')}" + f"-{time.time()}.json", + json.dumps( + jsonable_encoder(workflow_node_executions), + ).encode("utf-8"), + ) - workflow_node_execution_ids = [ - workflow_node_execution.id for workflow_node_execution in workflow_node_executions - ] + # Extract IDs for deletion + workflow_node_execution_ids = [ + workflow_node_execution.id for workflow_node_execution in workflow_node_executions + ] - # delete workflow node executions - session.query(WorkflowNodeExecutionModel).filter( - WorkflowNodeExecutionModel.id.in_(workflow_node_execution_ids), - ).delete(synchronize_session=False) - session.commit() + # Delete the backed up executions + deleted_count = node_execution_repo.delete_executions_by_ids(workflow_node_execution_ids) + total_deleted += deleted_count - click.echo( - click.style( - f"[{datetime.datetime.now()}] Processed {len(workflow_node_execution_ids)}" - f" workflow node executions for tenant {tenant_id}" - ) + click.echo( + click.style( + f"[{datetime.datetime.now()}] Processed {len(workflow_node_execution_ids)}" + f" workflow node executions for tenant {tenant_id}" ) + ) + + # If we got fewer than the batch size, we're done + if len(workflow_node_executions) < batch: + break + + # Process expired workflow runs with backup + session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + workflow_run_repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker) + before_date = datetime.datetime.now() - datetime.timedelta(days=days) + total_deleted = 0 while True: - with Session(db.engine).no_autoflush as session: - workflow_runs = ( - session.query(WorkflowRun) - .filter( - WorkflowRun.tenant_id == tenant_id, - WorkflowRun.created_at < datetime.datetime.now() - datetime.timedelta(days=days), - ) - .limit(batch) - .all() - ) + # Get a batch of expired workflow runs for backup + workflow_runs = workflow_run_repo.get_expired_runs_batch( + tenant_id=tenant_id, + before_date=before_date, + batch_size=batch, + ) - if len(workflow_runs) == 0: - break + if len(workflow_runs) == 0: + break + + # Save workflow runs to storage + storage.save( + f"free_plan_tenant_expired_logs/" + f"{tenant_id}/workflow_runs/{datetime.datetime.now().strftime('%Y-%m-%d')}" + f"-{time.time()}.json", + json.dumps( + jsonable_encoder( + [workflow_run.to_dict() for workflow_run in workflow_runs], + ), + ).encode("utf-8"), + ) - # save workflow runs + # Extract IDs for deletion + workflow_run_ids = [workflow_run.id for workflow_run in workflow_runs] - storage.save( - f"free_plan_tenant_expired_logs/" - f"{tenant_id}/workflow_runs/{datetime.datetime.now().strftime('%Y-%m-%d')}" - f"-{time.time()}.json", - json.dumps( - jsonable_encoder( - [workflow_run.to_dict() for workflow_run in workflow_runs], - ), - ).encode("utf-8"), - ) + # Delete the backed up workflow runs + deleted_count = workflow_run_repo.delete_runs_by_ids(workflow_run_ids) + total_deleted += deleted_count - workflow_run_ids = [workflow_run.id for workflow_run in workflow_runs] + click.echo( + click.style( + f"[{datetime.datetime.now()}] Processed {len(workflow_run_ids)}" + f" workflow runs for tenant {tenant_id}" + ) + ) - # delete workflow runs - session.query(WorkflowRun).filter( - WorkflowRun.id.in_(workflow_run_ids), - ).delete(synchronize_session=False) - session.commit() + # If we got fewer than the batch size, we're done + if len(workflow_runs) < batch: + break @classmethod def process(cls, days: int, batch: int, tenant_ids: list[str]): diff --git a/api/services/enterprise/enterprise_service.py b/api/services/enterprise/enterprise_service.py index 8c06ee9386..54d45f45ea 100644 --- a/api/services/enterprise/enterprise_service.py +++ b/api/services/enterprise/enterprise_service.py @@ -29,7 +29,7 @@ class EnterpriseService: raise ValueError("No data found.") try: # parse the UTC timestamp from the response - return datetime.fromisoformat(data.replace("Z", "+00:00")) + return datetime.fromisoformat(data) except ValueError as e: raise ValueError(f"Invalid date format: {data}") from e @@ -40,7 +40,7 @@ class EnterpriseService: raise ValueError("No data found.") try: # parse the UTC timestamp from the response - return datetime.fromisoformat(data.replace("Z", "+00:00")) + return datetime.fromisoformat(data) except ValueError as e: raise ValueError(f"Invalid date format: {data}") from e diff --git a/api/services/entities/knowledge_entities/knowledge_entities.py b/api/services/entities/knowledge_entities/knowledge_entities.py index 603064ca07..88d4224e97 100644 --- a/api/services/entities/knowledge_entities/knowledge_entities.py +++ b/api/services/entities/knowledge_entities/knowledge_entities.py @@ -95,7 +95,7 @@ class WeightKeywordSetting(BaseModel): class WeightModel(BaseModel): - weight_type: Optional[str] = None + weight_type: Optional[Literal["semantic_first", "keyword_first", "customized"]] = None vector_setting: Optional[WeightVectorSetting] = None keyword_setting: Optional[WeightKeywordSetting] = None diff --git a/api/services/file_service.py b/api/services/file_service.py index 2d68f30c5a..286535bd18 100644 --- a/api/services/file_service.py +++ b/api/services/file_service.py @@ -18,6 +18,7 @@ from core.file import helpers as file_helpers from core.rag.extractor.extract_processor import ExtractProcessor from extensions.ext_database import db from extensions.ext_storage import storage +from libs.helper import extract_tenant_id from models.account import Account from models.enums import CreatorUserRole from models.model import EndUser, UploadFile @@ -61,11 +62,7 @@ class FileService: # generate file key file_uuid = str(uuid.uuid4()) - if isinstance(user, Account): - current_tenant_id = user.current_tenant_id - else: - # end_user - current_tenant_id = user.tenant_id + current_tenant_id = extract_tenant_id(user) file_key = "upload_files/" + (current_tenant_id or "") + "/" + file_uuid + "." + extension diff --git a/api/services/plugin/plugin_service.py b/api/services/plugin/plugin_service.py index d7fb4a7c1b..0f22afd8dd 100644 --- a/api/services/plugin/plugin_service.py +++ b/api/services/plugin/plugin_service.py @@ -427,6 +427,9 @@ class PluginService: manager = PluginInstaller() + # collect actual plugin_unique_identifiers + actual_plugin_unique_identifiers = [] + metas = [] features = FeatureService.get_system_features() # check if already downloaded @@ -437,6 +440,8 @@ class PluginService: # check if the plugin is available to install PluginService._check_plugin_installation_scope(plugin_decode_response.verification) # already downloaded, skip + actual_plugin_unique_identifiers.append(plugin_unique_identifier) + metas.append({"plugin_unique_identifier": plugin_unique_identifier}) except Exception: # plugin not installed, download and upload pkg pkg = download_plugin_pkg(plugin_unique_identifier) @@ -447,17 +452,15 @@ class PluginService: ) # check if the plugin is available to install PluginService._check_plugin_installation_scope(response.verification) + # use response plugin_unique_identifier + actual_plugin_unique_identifiers.append(response.unique_identifier) + metas.append({"plugin_unique_identifier": response.unique_identifier}) return manager.install_from_identifiers( tenant_id, - plugin_unique_identifiers, + actual_plugin_unique_identifiers, PluginInstallationSource.Marketplace, - [ - { - "plugin_unique_identifier": plugin_unique_identifier, - } - for plugin_unique_identifier in plugin_unique_identifiers - ], + metas, ) @staticmethod diff --git a/api/services/tools/mcp_tools_mange_service.py b/api/services/tools/mcp_tools_mange_service.py new file mode 100644 index 0000000000..7c23abda4b --- /dev/null +++ b/api/services/tools/mcp_tools_mange_service.py @@ -0,0 +1,231 @@ +import hashlib +import json +from datetime import datetime +from typing import Any + +from sqlalchemy import or_ +from sqlalchemy.exc import IntegrityError + +from core.helper import encrypter +from core.mcp.error import MCPAuthError, MCPError +from core.mcp.mcp_client import MCPClient +from core.tools.entities.api_entities import ToolProviderApiEntity +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolProviderType +from core.tools.mcp_tool.provider import MCPToolProviderController +from core.tools.utils.configuration import ProviderConfigEncrypter +from extensions.ext_database import db +from models.tools import MCPToolProvider +from services.tools.tools_transform_service import ToolTransformService + +UNCHANGED_SERVER_URL_PLACEHOLDER = "[__HIDDEN__]" + + +class MCPToolManageService: + """ + Service class for managing mcp tools. + """ + + @staticmethod + def get_mcp_provider_by_provider_id(provider_id: str, tenant_id: str) -> MCPToolProvider: + res = ( + db.session.query(MCPToolProvider) + .filter(MCPToolProvider.tenant_id == tenant_id, MCPToolProvider.id == provider_id) + .first() + ) + if not res: + raise ValueError("MCP tool not found") + return res + + @staticmethod + def get_mcp_provider_by_server_identifier(server_identifier: str, tenant_id: str) -> MCPToolProvider: + res = ( + db.session.query(MCPToolProvider) + .filter(MCPToolProvider.tenant_id == tenant_id, MCPToolProvider.server_identifier == server_identifier) + .first() + ) + if not res: + raise ValueError("MCP tool not found") + return res + + @staticmethod + def create_mcp_provider( + tenant_id: str, + name: str, + server_url: str, + user_id: str, + icon: str, + icon_type: str, + icon_background: str, + server_identifier: str, + ) -> ToolProviderApiEntity: + server_url_hash = hashlib.sha256(server_url.encode()).hexdigest() + existing_provider = ( + db.session.query(MCPToolProvider) + .filter( + MCPToolProvider.tenant_id == tenant_id, + or_( + MCPToolProvider.name == name, + MCPToolProvider.server_url_hash == server_url_hash, + MCPToolProvider.server_identifier == server_identifier, + ), + ) + .first() + ) + if existing_provider: + if existing_provider.name == name: + raise ValueError(f"MCP tool {name} already exists") + elif existing_provider.server_url_hash == server_url_hash: + raise ValueError(f"MCP tool {server_url} already exists") + elif existing_provider.server_identifier == server_identifier: + raise ValueError(f"MCP tool {server_identifier} already exists") + encrypted_server_url = encrypter.encrypt_token(tenant_id, server_url) + mcp_tool = MCPToolProvider( + tenant_id=tenant_id, + name=name, + server_url=encrypted_server_url, + server_url_hash=server_url_hash, + user_id=user_id, + authed=False, + tools="[]", + icon=json.dumps({"content": icon, "background": icon_background}) if icon_type == "emoji" else icon, + server_identifier=server_identifier, + ) + db.session.add(mcp_tool) + db.session.commit() + return ToolTransformService.mcp_provider_to_user_provider(mcp_tool, for_list=True) + + @staticmethod + def retrieve_mcp_tools(tenant_id: str, for_list: bool = False) -> list[ToolProviderApiEntity]: + mcp_providers = ( + db.session.query(MCPToolProvider) + .filter(MCPToolProvider.tenant_id == tenant_id) + .order_by(MCPToolProvider.name) + .all() + ) + return [ + ToolTransformService.mcp_provider_to_user_provider(mcp_provider, for_list=for_list) + for mcp_provider in mcp_providers + ] + + @classmethod + def list_mcp_tool_from_remote_server(cls, tenant_id: str, provider_id: str): + mcp_provider = cls.get_mcp_provider_by_provider_id(provider_id, tenant_id) + + try: + with MCPClient( + mcp_provider.decrypted_server_url, provider_id, tenant_id, authed=mcp_provider.authed, for_list=True + ) as mcp_client: + tools = mcp_client.list_tools() + except MCPAuthError as e: + raise ValueError("Please auth the tool first") + except MCPError as e: + raise ValueError(f"Failed to connect to MCP server: {e}") + mcp_provider.tools = json.dumps([tool.model_dump() for tool in tools]) + mcp_provider.authed = True + mcp_provider.updated_at = datetime.now() + db.session.commit() + user = mcp_provider.load_user() + return ToolProviderApiEntity( + id=mcp_provider.id, + name=mcp_provider.name, + tools=ToolTransformService.mcp_tool_to_user_tool(mcp_provider, tools), + type=ToolProviderType.MCP, + icon=mcp_provider.icon, + author=user.name if user else "Anonymous", + server_url=mcp_provider.masked_server_url, + updated_at=int(mcp_provider.updated_at.timestamp()), + description=I18nObject(en_US="", zh_Hans=""), + label=I18nObject(en_US=mcp_provider.name, zh_Hans=mcp_provider.name), + plugin_unique_identifier=mcp_provider.server_identifier, + ) + + @classmethod + def delete_mcp_tool(cls, tenant_id: str, provider_id: str): + mcp_tool = cls.get_mcp_provider_by_provider_id(provider_id, tenant_id) + + db.session.delete(mcp_tool) + db.session.commit() + + @classmethod + def update_mcp_provider( + cls, + tenant_id: str, + provider_id: str, + name: str, + server_url: str, + icon: str, + icon_type: str, + icon_background: str, + server_identifier: str, + ): + mcp_provider = cls.get_mcp_provider_by_provider_id(provider_id, tenant_id) + mcp_provider.updated_at = datetime.now() + mcp_provider.name = name + mcp_provider.icon = ( + json.dumps({"content": icon, "background": icon_background}) if icon_type == "emoji" else icon + ) + mcp_provider.server_identifier = server_identifier + + if UNCHANGED_SERVER_URL_PLACEHOLDER not in server_url: + encrypted_server_url = encrypter.encrypt_token(tenant_id, server_url) + mcp_provider.server_url = encrypted_server_url + server_url_hash = hashlib.sha256(server_url.encode()).hexdigest() + + if server_url_hash != mcp_provider.server_url_hash: + cls._re_connect_mcp_provider(mcp_provider, provider_id, tenant_id) + mcp_provider.server_url_hash = server_url_hash + try: + db.session.commit() + except IntegrityError as e: + db.session.rollback() + error_msg = str(e.orig) + if "unique_mcp_provider_name" in error_msg: + raise ValueError(f"MCP tool {name} already exists") + elif "unique_mcp_provider_server_url" in error_msg: + raise ValueError(f"MCP tool {server_url} already exists") + elif "unique_mcp_provider_server_identifier" in error_msg: + raise ValueError(f"MCP tool {server_identifier} already exists") + else: + raise + + @classmethod + def update_mcp_provider_credentials( + cls, mcp_provider: MCPToolProvider, credentials: dict[str, Any], authed: bool = False + ): + provider_controller = MCPToolProviderController._from_db(mcp_provider) + tool_configuration = ProviderConfigEncrypter( + tenant_id=mcp_provider.tenant_id, + config=list(provider_controller.get_credentials_schema()), + provider_type=provider_controller.provider_type.value, + provider_identity=provider_controller.provider_id, + ) + credentials = tool_configuration.encrypt(credentials) + mcp_provider.updated_at = datetime.now() + mcp_provider.encrypted_credentials = json.dumps({**mcp_provider.credentials, **credentials}) + mcp_provider.authed = authed + if not authed: + mcp_provider.tools = "[]" + db.session.commit() + + @classmethod + def _re_connect_mcp_provider(cls, mcp_provider: MCPToolProvider, provider_id: str, tenant_id: str): + """re-connect mcp provider""" + try: + with MCPClient( + mcp_provider.decrypted_server_url, + provider_id, + tenant_id, + authed=False, + for_list=True, + ) as mcp_client: + tools = mcp_client.list_tools() + mcp_provider.authed = True + mcp_provider.tools = json.dumps([tool.model_dump() for tool in tools]) + except MCPAuthError: + mcp_provider.authed = False + mcp_provider.tools = "[]" + except MCPError as e: + raise ValueError(f"Failed to re-connect MCP server: {e}") from e + # reset credentials + mcp_provider.encrypted_credentials = "{}" diff --git a/api/services/tools/tools_transform_service.py b/api/services/tools/tools_transform_service.py index 367121125b..3d0c35cd9b 100644 --- a/api/services/tools/tools_transform_service.py +++ b/api/services/tools/tools_transform_service.py @@ -1,10 +1,11 @@ import json import logging -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from yarl import URL from configs import dify_config +from core.mcp.types import Tool as MCPTool from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime from core.tools.builtin_tool.provider import BuiltinToolProviderController @@ -21,7 +22,7 @@ from core.tools.plugin_tool.provider import PluginToolProviderController from core.tools.utils.configuration import ProviderConfigEncrypter from core.tools.workflow_as_tool.provider import WorkflowToolProviderController from core.tools.workflow_as_tool.tool import WorkflowTool -from models.tools import ApiToolProvider, BuiltinToolProvider, WorkflowToolProvider +from models.tools import ApiToolProvider, BuiltinToolProvider, MCPToolProvider, WorkflowToolProvider logger = logging.getLogger(__name__) @@ -52,7 +53,8 @@ class ToolTransformService: return icon except Exception: return {"background": "#252525", "content": "\ud83d\ude01"} - + elif provider_type == ToolProviderType.MCP.value: + return icon return "" @staticmethod @@ -73,10 +75,18 @@ class ToolTransformService: provider.icon = ToolTransformService.get_plugin_icon_url( tenant_id=tenant_id, filename=provider.icon ) + if isinstance(provider.icon_dark, str) and provider.icon_dark: + provider.icon_dark = ToolTransformService.get_plugin_icon_url( + tenant_id=tenant_id, filename=provider.icon_dark + ) else: provider.icon = ToolTransformService.get_tool_provider_icon_url( provider_type=provider.type.value, provider_name=provider.name, icon=provider.icon ) + if provider.icon_dark: + provider.icon_dark = ToolTransformService.get_tool_provider_icon_url( + provider_type=provider.type.value, provider_name=provider.name, icon=provider.icon_dark + ) @classmethod def builtin_provider_to_user_provider( @@ -94,6 +104,7 @@ class ToolTransformService: name=provider_controller.entity.identity.name, description=provider_controller.entity.identity.description, icon=provider_controller.entity.identity.icon, + icon_dark=provider_controller.entity.identity.icon_dark, label=provider_controller.entity.identity.label, type=ToolProviderType.BUILT_IN, masked_credentials={}, @@ -148,11 +159,16 @@ class ToolTransformService: convert provider controller to user provider """ # package tool provider controller + auth_type = ApiProviderAuthType.NONE + credentials_auth_type = db_provider.credentials.get("auth_type") + if credentials_auth_type in ("api_key_header", "api_key"): # backward compatibility + auth_type = ApiProviderAuthType.API_KEY_HEADER + elif credentials_auth_type == "api_key_query": + auth_type = ApiProviderAuthType.API_KEY_QUERY + controller = ApiToolProviderController.from_db( db_provider=db_provider, - auth_type=ApiProviderAuthType.API_KEY - if db_provider.credentials["auth_type"] == "api_key" - else ApiProviderAuthType.NONE, + auth_type=auth_type, ) return controller @@ -177,6 +193,7 @@ class ToolTransformService: name=provider_controller.entity.identity.name, description=provider_controller.entity.identity.description, icon=provider_controller.entity.identity.icon, + icon_dark=provider_controller.entity.identity.icon_dark, label=provider_controller.entity.identity.label, type=ToolProviderType.WORKFLOW, masked_credentials={}, @@ -187,6 +204,41 @@ class ToolTransformService: labels=labels or [], ) + @staticmethod + def mcp_provider_to_user_provider(db_provider: MCPToolProvider, for_list: bool = False) -> ToolProviderApiEntity: + user = db_provider.load_user() + return ToolProviderApiEntity( + id=db_provider.server_identifier if not for_list else db_provider.id, + author=user.name if user else "Anonymous", + name=db_provider.name, + icon=db_provider.provider_icon, + type=ToolProviderType.MCP, + is_team_authorization=db_provider.authed, + server_url=db_provider.masked_server_url, + tools=ToolTransformService.mcp_tool_to_user_tool( + db_provider, [MCPTool(**tool) for tool in json.loads(db_provider.tools)] + ), + updated_at=int(db_provider.updated_at.timestamp()), + label=I18nObject(en_US=db_provider.name, zh_Hans=db_provider.name), + description=I18nObject(en_US="", zh_Hans=""), + server_identifier=db_provider.server_identifier, + ) + + @staticmethod + def mcp_tool_to_user_tool(mcp_provider: MCPToolProvider, tools: list[MCPTool]) -> list[ToolApiEntity]: + user = mcp_provider.load_user() + return [ + ToolApiEntity( + author=user.name if user else "Anonymous", + name=tool.name, + label=I18nObject(en_US=tool.name, zh_Hans=tool.name), + description=I18nObject(en_US=tool.description, zh_Hans=tool.description), + parameters=ToolTransformService.convert_mcp_schema_to_parameter(tool.inputSchema), + labels=[], + ) + for tool in tools + ] + @classmethod def api_provider_to_user_provider( cls, @@ -304,3 +356,53 @@ class ToolTransformService: parameters=tool.parameters, labels=labels or [], ) + + @staticmethod + def convert_mcp_schema_to_parameter(schema: dict) -> list["ToolParameter"]: + """ + Convert MCP JSON schema to tool parameters + + :param schema: JSON schema dictionary + :return: list of ToolParameter instances + """ + + def create_parameter( + name: str, description: str, param_type: str, required: bool, input_schema: dict | None = None + ) -> ToolParameter: + """Create a ToolParameter instance with given attributes""" + input_schema_dict: dict[str, Any] = {"input_schema": input_schema} if input_schema else {} + return ToolParameter( + name=name, + llm_description=description, + label=I18nObject(en_US=name), + form=ToolParameter.ToolParameterForm.LLM, + required=required, + type=ToolParameter.ToolParameterType(param_type), + human_description=I18nObject(en_US=description), + **input_schema_dict, + ) + + def process_properties(props: dict, required: list, prefix: str = "") -> list[ToolParameter]: + """Process properties recursively""" + TYPE_MAPPING = {"integer": "number", "float": "number"} + COMPLEX_TYPES = ["array", "object"] + + parameters = [] + for name, prop in props.items(): + current_description = prop.get("description", "") + prop_type = prop.get("type", "string") + + if isinstance(prop_type, list): + prop_type = prop_type[0] + if prop_type in TYPE_MAPPING: + prop_type = TYPE_MAPPING[prop_type] + input_schema = prop if prop_type in COMPLEX_TYPES else None + parameters.append( + create_parameter(name, current_description, prop_type, name in required, input_schema) + ) + + return parameters + + if schema.get("type") == "object" and "properties" in schema: + return process_properties(schema["properties"], schema.get("required", [])) + return [] diff --git a/api/services/workflow_draft_variable_service.py b/api/services/workflow_draft_variable_service.py index 44fd72b5e4..f306e1f062 100644 --- a/api/services/workflow_draft_variable_service.py +++ b/api/services/workflow_draft_variable_service.py @@ -5,9 +5,9 @@ from collections.abc import Mapping, Sequence from enum import StrEnum from typing import Any, ClassVar -from sqlalchemy import Engine, orm, select +from sqlalchemy import Engine, orm from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.sql.expression import and_, or_ from core.app.entities.app_invoke_entities import InvokeFrom @@ -25,7 +25,8 @@ from factories.file_factory import StorageKeyLoader from factories.variable_factory import build_segment, segment_to_variable from models import App, Conversation from models.enums import DraftVariableType -from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel, is_system_variable_editable +from models.workflow import Workflow, WorkflowDraftVariable, is_system_variable_editable +from repositories.factory import DifyAPIRepositoryFactory _logger = logging.getLogger(__name__) @@ -117,7 +118,24 @@ class WorkflowDraftVariableService: _session: Session def __init__(self, session: Session) -> None: + """ + Initialize the WorkflowDraftVariableService with a SQLAlchemy session. + + Args: + session (Session): The SQLAlchemy session used to execute database queries. + The provided session must be bound to an `Engine` object, not a specific `Connection`. + + Raises: + AssertionError: If the provided session is not bound to an `Engine` object. + """ self._session = session + engine = session.get_bind() + # Ensure the session is bound to a engine. + assert isinstance(engine, Engine) + session_maker = sessionmaker(bind=engine, expire_on_commit=False) + self._api_node_execution_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( + session_maker + ) def get_variable(self, variable_id: str) -> WorkflowDraftVariable | None: return self._session.query(WorkflowDraftVariable).filter(WorkflowDraftVariable.id == variable_id).first() @@ -248,8 +266,7 @@ class WorkflowDraftVariableService: _logger.warning("draft variable has no node_execution_id, id=%s, name=%s", variable.id, variable.name) return None - query = select(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.id == variable.node_execution_id) - node_exec = self._session.scalars(query).first() + node_exec = self._api_node_execution_repo.get_execution_by_id(variable.node_execution_id) if node_exec is None: _logger.warning( "Node exectution not found for draft variable, id=%s, name=%s, node_execution_id=%s", @@ -298,6 +315,8 @@ class WorkflowDraftVariableService: def reset_variable(self, workflow: Workflow, variable: WorkflowDraftVariable) -> WorkflowDraftVariable | None: variable_type = variable.get_variable_type() + if variable_type == DraftVariableType.SYS and not is_system_variable_editable(variable.name): + raise VariableResetError(f"cannot reset system variable, variable_id={variable.id}") if variable_type == DraftVariableType.CONVERSATION: return self._reset_conv_var(workflow, variable) else: diff --git a/api/services/workflow_run_service.py b/api/services/workflow_run_service.py index 483c0d3086..e43999a8c9 100644 --- a/api/services/workflow_run_service.py +++ b/api/services/workflow_run_service.py @@ -2,9 +2,9 @@ import threading from collections.abc import Sequence from typing import Optional +from sqlalchemy.orm import sessionmaker + import contexts -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository -from core.workflow.repositories.workflow_node_execution_repository import OrderConfig from extensions.ext_database import db from libs.infinite_scroll_pagination import InfiniteScrollPagination from models import ( @@ -15,10 +15,18 @@ from models import ( WorkflowRun, WorkflowRunTriggeredFrom, ) -from models.workflow import WorkflowNodeExecutionTriggeredFrom +from repositories.factory import DifyAPIRepositoryFactory class WorkflowRunService: + def __init__(self): + """Initialize WorkflowRunService with repository dependencies.""" + session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( + session_maker + ) + self._workflow_run_repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker) + def get_paginate_advanced_chat_workflow_runs(self, app_model: App, args: dict) -> InfiniteScrollPagination: """ Get advanced chat app workflow run list @@ -62,45 +70,16 @@ class WorkflowRunService: :param args: request args """ limit = int(args.get("limit", 20)) + last_id = args.get("last_id") - base_query = db.session.query(WorkflowRun).filter( - WorkflowRun.tenant_id == app_model.tenant_id, - WorkflowRun.app_id == app_model.id, - WorkflowRun.triggered_from == WorkflowRunTriggeredFrom.DEBUGGING.value, + return self._workflow_run_repo.get_paginated_workflow_runs( + tenant_id=app_model.tenant_id, + app_id=app_model.id, + triggered_from=WorkflowRunTriggeredFrom.DEBUGGING.value, + limit=limit, + last_id=last_id, ) - if args.get("last_id"): - last_workflow_run = base_query.filter( - WorkflowRun.id == args.get("last_id"), - ).first() - - if not last_workflow_run: - raise ValueError("Last workflow run not exists") - - workflow_runs = ( - base_query.filter( - WorkflowRun.created_at < last_workflow_run.created_at, WorkflowRun.id != last_workflow_run.id - ) - .order_by(WorkflowRun.created_at.desc()) - .limit(limit) - .all() - ) - else: - workflow_runs = base_query.order_by(WorkflowRun.created_at.desc()).limit(limit).all() - - has_more = False - if len(workflow_runs) == limit: - current_page_first_workflow_run = workflow_runs[-1] - rest_count = base_query.filter( - WorkflowRun.created_at < current_page_first_workflow_run.created_at, - WorkflowRun.id != current_page_first_workflow_run.id, - ).count() - - if rest_count > 0: - has_more = True - - return InfiniteScrollPagination(data=workflow_runs, limit=limit, has_more=has_more) - def get_workflow_run(self, app_model: App, run_id: str) -> Optional[WorkflowRun]: """ Get workflow run detail @@ -108,18 +87,12 @@ class WorkflowRunService: :param app_model: app model :param run_id: workflow run id """ - workflow_run = ( - db.session.query(WorkflowRun) - .filter( - WorkflowRun.tenant_id == app_model.tenant_id, - WorkflowRun.app_id == app_model.id, - WorkflowRun.id == run_id, - ) - .first() + return self._workflow_run_repo.get_workflow_run_by_id( + tenant_id=app_model.tenant_id, + app_id=app_model.id, + run_id=run_id, ) - return workflow_run - def get_workflow_run_node_executions( self, app_model: App, @@ -137,17 +110,13 @@ class WorkflowRunService: if not workflow_run: return [] - repository = SQLAlchemyWorkflowNodeExecutionRepository( - session_factory=db.engine, - user=user, - app_id=app_model.id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, - ) + # Get tenant_id from user + tenant_id = user.tenant_id if isinstance(user, EndUser) else user.current_tenant_id + if tenant_id is None: + raise ValueError("User tenant_id cannot be None") - # Use the repository to get the database models directly - order_config = OrderConfig(order_by=["index"], order_direction="desc") - workflow_node_executions = repository.get_db_models_by_workflow_run( - workflow_run_id=run_id, order_config=order_config + return self._node_execution_service_repo.get_executions_by_workflow_run( + tenant_id=tenant_id, + app_id=app_model.id, + workflow_run_id=run_id, ) - - return workflow_node_executions diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 2be57fd51c..0149d50346 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -7,13 +7,13 @@ from typing import Any, Optional from uuid import uuid4 from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.app.app_config.entities import VariableEntityType from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.file import File -from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository +from core.repositories import DifyCoreRepositoryFactory from core.variables import Variable from core.workflow.entities.node_entities import NodeRunResult from core.workflow.entities.variable_pool import VariablePool @@ -41,6 +41,7 @@ from models.workflow import ( WorkflowNodeExecutionTriggeredFrom, WorkflowType, ) +from repositories.factory import DifyAPIRepositoryFactory from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError from services.workflow.workflow_converter import WorkflowConverter @@ -57,21 +58,32 @@ class WorkflowService: Workflow Service """ - def get_node_last_run(self, app_model: App, workflow: Workflow, node_id: str) -> WorkflowNodeExecutionModel | None: - # TODO(QuantumGhost): This query is not fully covered by index. - criteria = ( - WorkflowNodeExecutionModel.tenant_id == app_model.tenant_id, - WorkflowNodeExecutionModel.app_id == app_model.id, - WorkflowNodeExecutionModel.workflow_id == workflow.id, - WorkflowNodeExecutionModel.node_id == node_id, + def __init__(self, session_maker: sessionmaker | None = None): + """Initialize WorkflowService with repository dependencies.""" + if session_maker is None: + session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( + session_maker ) - node_exec = ( - db.session.query(WorkflowNodeExecutionModel) - .filter(*criteria) - .order_by(WorkflowNodeExecutionModel.created_at.desc()) - .first() + + def get_node_last_run(self, app_model: App, workflow: Workflow, node_id: str) -> WorkflowNodeExecutionModel | None: + """ + Get the most recent execution for a specific node. + + Args: + app_model: The application model + workflow: The workflow model + node_id: The node identifier + + Returns: + The most recent WorkflowNodeExecutionModel for the node, or None if not found + """ + return self._node_execution_service_repo.get_node_last_execution( + tenant_id=app_model.tenant_id, + app_id=app_model.id, + workflow_id=workflow.id, + node_id=node_id, ) - return node_exec def is_workflow_exist(self, app_model: App) -> bool: return ( @@ -396,7 +408,7 @@ class WorkflowService: node_execution.workflow_id = draft_workflow.id # Create repository and save the node execution - repository = SQLAlchemyWorkflowNodeExecutionRepository( + repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( session_factory=db.engine, user=account, app_id=app_model.id, @@ -404,8 +416,9 @@ class WorkflowService: ) repository.save(node_execution) - # Convert node_execution to WorkflowNodeExecution after save - workflow_node_execution = repository.to_db_model(node_execution) + workflow_node_execution = self._node_execution_service_repo.get_execution_by_id(node_execution.id) + if workflow_node_execution is None: + raise ValueError(f"WorkflowNodeExecution with id {node_execution.id} not found after saving") with Session(bind=db.engine) as session, session.begin(): draft_var_saver = DraftVariableSaver( @@ -418,6 +431,7 @@ class WorkflowService: ) draft_var_saver.save(process_data=node_execution.process_data, outputs=node_execution.outputs) session.commit() + return workflow_node_execution def run_free_workflow_node( @@ -429,7 +443,7 @@ class WorkflowService: # run draft workflow node start_at = time.perf_counter() - workflow_node_execution = self._handle_node_run_result( + node_execution = self._handle_node_run_result( invoke_node_fn=lambda: WorkflowEntry.run_free_node( node_id=node_id, node_data=node_data, @@ -441,7 +455,7 @@ class WorkflowService: node_id=node_id, ) - return workflow_node_execution + return node_execution def _handle_node_run_result( self, diff --git a/api/tasks/clean_document_task.py b/api/tasks/clean_document_task.py index 5824121e8f..c72a3319c1 100644 --- a/api/tasks/clean_document_task.py +++ b/api/tasks/clean_document_task.py @@ -72,6 +72,7 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_i DatasetMetadataBinding.dataset_id == dataset_id, DatasetMetadataBinding.document_id == document_id, ).delete() + db.session.commit() end_at = time.perf_counter() logging.info( diff --git a/api/tasks/remove_app_and_related_data_task.py b/api/tasks/remove_app_and_related_data_task.py index d366efd6f2..179adcbd6e 100644 --- a/api/tasks/remove_app_and_related_data_task.py +++ b/api/tasks/remove_app_and_related_data_task.py @@ -6,6 +6,7 @@ import click from celery import shared_task # type: ignore from sqlalchemy import delete from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import sessionmaker from extensions.ext_database import db from models import ( @@ -13,6 +14,7 @@ from models import ( AppAnnotationHitHistory, AppAnnotationSetting, AppDatasetJoin, + AppMCPServer, AppModelConfig, Conversation, EndUser, @@ -30,7 +32,8 @@ from models import ( ) from models.tools import WorkflowToolProvider from models.web import PinnedConversation, SavedMessage -from models.workflow import ConversationVariable, Workflow, WorkflowAppLog, WorkflowNodeExecutionModel, WorkflowRun +from models.workflow import ConversationVariable, Workflow, WorkflowAppLog +from repositories.factory import DifyAPIRepositoryFactory @shared_task(queue="app_deletion", bind=True, max_retries=3) @@ -41,6 +44,7 @@ def remove_app_and_related_data_task(self, tenant_id: str, app_id: str): # Delete related data _delete_app_model_configs(tenant_id, app_id) _delete_app_site(tenant_id, app_id) + _delete_app_mcp_servers(tenant_id, app_id) _delete_app_api_tokens(tenant_id, app_id) _delete_installed_apps(tenant_id, app_id) _delete_recommended_apps(tenant_id, app_id) @@ -89,6 +93,18 @@ def _delete_app_site(tenant_id: str, app_id: str): _delete_records("""select id from sites where app_id=:app_id limit 1000""", {"app_id": app_id}, del_site, "site") +def _delete_app_mcp_servers(tenant_id: str, app_id: str): + def del_mcp_server(mcp_server_id: str): + db.session.query(AppMCPServer).filter(AppMCPServer.id == mcp_server_id).delete(synchronize_session=False) + + _delete_records( + """select id from app_mcp_servers where app_id=:app_id limit 1000""", + {"app_id": app_id}, + del_mcp_server, + "app mcp server", + ) + + def _delete_app_api_tokens(tenant_id: str, app_id: str): def del_api_token(api_token_id: str): db.session.query(ApiToken).filter(ApiToken.id == api_token_id).delete(synchronize_session=False) @@ -175,30 +191,32 @@ def _delete_app_workflows(tenant_id: str, app_id: str): def _delete_app_workflow_runs(tenant_id: str, app_id: str): - def del_workflow_run(workflow_run_id: str): - db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_run_id).delete(synchronize_session=False) - - _delete_records( - """select id from workflow_runs where tenant_id=:tenant_id and app_id=:app_id limit 1000""", - {"tenant_id": tenant_id, "app_id": app_id}, - del_workflow_run, - "workflow run", + """Delete all workflow runs for an app using the service repository.""" + session_maker = sessionmaker(bind=db.engine) + workflow_run_repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker) + + deleted_count = workflow_run_repo.delete_runs_by_app( + tenant_id=tenant_id, + app_id=app_id, + batch_size=1000, ) + logging.info(f"Deleted {deleted_count} workflow runs for app {app_id}") -def _delete_app_workflow_node_executions(tenant_id: str, app_id: str): - def del_workflow_node_execution(workflow_node_execution_id: str): - db.session.query(WorkflowNodeExecutionModel).filter( - WorkflowNodeExecutionModel.id == workflow_node_execution_id - ).delete(synchronize_session=False) - _delete_records( - """select id from workflow_node_executions where tenant_id=:tenant_id and app_id=:app_id limit 1000""", - {"tenant_id": tenant_id, "app_id": app_id}, - del_workflow_node_execution, - "workflow node execution", +def _delete_app_workflow_node_executions(tenant_id: str, app_id: str): + """Delete all workflow node executions for an app using the service repository.""" + session_maker = sessionmaker(bind=db.engine) + node_execution_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(session_maker) + + deleted_count = node_execution_repo.delete_executions_by_app( + tenant_id=tenant_id, + app_id=app_id, + batch_size=1000, ) + logging.info(f"Deleted {deleted_count} workflow node executions for app {app_id}") + def _delete_app_workflow_app_logs(tenant_id: str, app_id: str): def del_workflow_app_log(workflow_app_log_id: str): diff --git a/api/tests/integration_tests/vdb/couchbase/test_couchbase.py b/api/tests/integration_tests/vdb/couchbase/test_couchbase.py index d76c34ba0e..eef1ee4e75 100644 --- a/api/tests/integration_tests/vdb/couchbase/test_couchbase.py +++ b/api/tests/integration_tests/vdb/couchbase/test_couchbase.py @@ -4,7 +4,6 @@ import time from core.rag.datasource.vdb.couchbase.couchbase_vector import CouchbaseConfig, CouchbaseVector from tests.integration_tests.vdb.test_vector_store import ( AbstractVectorTest, - get_example_text, setup_mock_redis, ) diff --git a/api/tests/integration_tests/vdb/matrixone/test_matrixone.py b/api/tests/integration_tests/vdb/matrixone/test_matrixone.py index c8b19ef3ad..c4056db63e 100644 --- a/api/tests/integration_tests/vdb/matrixone/test_matrixone.py +++ b/api/tests/integration_tests/vdb/matrixone/test_matrixone.py @@ -1,7 +1,6 @@ from core.rag.datasource.vdb.matrixone.matrixone_vector import MatrixoneConfig, MatrixoneVector from tests.integration_tests.vdb.test_vector_store import ( AbstractVectorTest, - get_example_text, setup_mock_redis, ) diff --git a/api/tests/integration_tests/vdb/opengauss/test_opengauss.py b/api/tests/integration_tests/vdb/opengauss/test_opengauss.py index f2013848bf..2a1129493c 100644 --- a/api/tests/integration_tests/vdb/opengauss/test_opengauss.py +++ b/api/tests/integration_tests/vdb/opengauss/test_opengauss.py @@ -5,7 +5,6 @@ import psycopg2 # type: ignore from core.rag.datasource.vdb.opengauss.opengauss import OpenGauss, OpenGaussConfig from tests.integration_tests.vdb.test_vector_store import ( AbstractVectorTest, - get_example_text, setup_mock_redis, ) diff --git a/api/tests/integration_tests/vdb/pyvastbase/test_vastbase_vector.py b/api/tests/integration_tests/vdb/pyvastbase/test_vastbase_vector.py index 3d7873442b..02931fef5a 100644 --- a/api/tests/integration_tests/vdb/pyvastbase/test_vastbase_vector.py +++ b/api/tests/integration_tests/vdb/pyvastbase/test_vastbase_vector.py @@ -1,7 +1,6 @@ from core.rag.datasource.vdb.pyvastbase.vastbase_vector import VastbaseVector, VastbaseVectorConfig from tests.integration_tests.vdb.test_vector_store import ( AbstractVectorTest, - get_example_text, setup_mock_redis, ) diff --git a/api/tests/integration_tests/workflow/nodes/test_llm.py b/api/tests/integration_tests/workflow/nodes/test_llm.py index 389d1071f3..8acaa54b9c 100644 --- a/api/tests/integration_tests/workflow/nodes/test_llm.py +++ b/api/tests/integration_tests/workflow/nodes/test_llm.py @@ -1,5 +1,4 @@ import json -import os import time import uuid from collections.abc import Generator @@ -113,17 +112,15 @@ def test_execute_llm(flask_req_ctx): }, ) - credentials = {"openai_api_key": os.environ.get("OPENAI_API_KEY")} - # Create a proper LLM result with real entities mock_usage = LLMUsage( prompt_tokens=30, prompt_unit_price=Decimal("0.001"), - prompt_price_unit=Decimal("1000"), + prompt_price_unit=Decimal(1000), prompt_price=Decimal("0.00003"), completion_tokens=20, completion_unit_price=Decimal("0.002"), - completion_price_unit=Decimal("1000"), + completion_price_unit=Decimal(1000), completion_price=Decimal("0.00004"), total_tokens=50, total_price=Decimal("0.00007"), @@ -222,11 +219,11 @@ def test_execute_llm_with_jinja2(flask_req_ctx, setup_code_executor_mock): mock_usage = LLMUsage( prompt_tokens=30, prompt_unit_price=Decimal("0.001"), - prompt_price_unit=Decimal("1000"), + prompt_price_unit=Decimal(1000), prompt_price=Decimal("0.00003"), completion_tokens=20, completion_unit_price=Decimal("0.002"), - completion_price_unit=Decimal("1000"), + completion_price_unit=Decimal(1000), completion_price=Decimal("0.00004"), total_tokens=50, total_price=Decimal("0.00007"), diff --git a/api/tests/unit_tests/core/helper/test_encrypter.py b/api/tests/unit_tests/core/helper/test_encrypter.py new file mode 100644 index 0000000000..61cf8f255d --- /dev/null +++ b/api/tests/unit_tests/core/helper/test_encrypter.py @@ -0,0 +1,280 @@ +import base64 +import binascii +from unittest.mock import MagicMock, patch + +import pytest + +from core.helper.encrypter import ( + batch_decrypt_token, + decrypt_token, + encrypt_token, + get_decrypt_decoding, + obfuscated_token, +) +from libs.rsa import PrivkeyNotFoundError + + +class TestObfuscatedToken: + @pytest.mark.parametrize( + ("token", "expected"), + [ + ("", ""), # Empty token + ("1234567", "*" * 20), # Short token (<8 chars) + ("12345678", "*" * 20), # Boundary case (8 chars) + ("123456789abcdef", "123456" + "*" * 12 + "ef"), # Long token + ("abc!@#$%^&*()def", "abc!@#" + "*" * 12 + "ef"), # Special chars + ], + ) + def test_obfuscation_logic(self, token, expected): + """Test core obfuscation logic for various token lengths""" + assert obfuscated_token(token) == expected + + def test_sensitive_data_protection(self): + """Ensure obfuscation never reveals full sensitive data""" + token = "api_key_secret_12345" + obfuscated = obfuscated_token(token) + assert token not in obfuscated + assert "*" * 12 in obfuscated + + +class TestEncryptToken: + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + def test_successful_encryption(self, mock_encrypt, mock_query): + """Test successful token encryption""" + mock_tenant = MagicMock() + mock_tenant.encrypt_public_key = "mock_public_key" + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + mock_encrypt.return_value = b"encrypted_data" + + result = encrypt_token("tenant-123", "test_token") + + assert result == base64.b64encode(b"encrypted_data").decode() + mock_encrypt.assert_called_with("test_token", "mock_public_key") + + @patch("models.engine.db.session.query") + def test_tenant_not_found(self, mock_query): + """Test error when tenant doesn't exist""" + mock_query.return_value.filter.return_value.first.return_value = None + + with pytest.raises(ValueError) as exc_info: + encrypt_token("invalid-tenant", "test_token") + + assert "Tenant with id invalid-tenant not found" in str(exc_info.value) + + +class TestDecryptToken: + @patch("libs.rsa.decrypt") + def test_successful_decryption(self, mock_decrypt): + """Test successful token decryption""" + mock_decrypt.return_value = "decrypted_token" + encrypted_data = base64.b64encode(b"encrypted_data").decode() + + result = decrypt_token("tenant-123", encrypted_data) + + assert result == "decrypted_token" + mock_decrypt.assert_called_once_with(b"encrypted_data", "tenant-123") + + def test_invalid_base64(self): + """Test handling of invalid base64 input""" + with pytest.raises(binascii.Error): + decrypt_token("tenant-123", "invalid_base64!!!") + + +class TestBatchDecryptToken: + @patch("libs.rsa.get_decrypt_decoding") + @patch("libs.rsa.decrypt_token_with_decoding") + def test_batch_decryption(self, mock_decrypt_with_decoding, mock_get_decoding): + """Test batch decryption functionality""" + mock_rsa_key = MagicMock() + mock_cipher_rsa = MagicMock() + mock_get_decoding.return_value = (mock_rsa_key, mock_cipher_rsa) + + # Test multiple tokens + mock_decrypt_with_decoding.side_effect = ["token1", "token2", "token3"] + tokens = [ + base64.b64encode(b"encrypted1").decode(), + base64.b64encode(b"encrypted2").decode(), + base64.b64encode(b"encrypted3").decode(), + ] + result = batch_decrypt_token("tenant-123", tokens) + + assert result == ["token1", "token2", "token3"] + # Key should only be loaded once + mock_get_decoding.assert_called_once_with("tenant-123") + + +class TestGetDecryptDecoding: + @patch("extensions.ext_redis.redis_client.get") + @patch("extensions.ext_storage.storage.load") + def test_private_key_not_found(self, mock_storage_load, mock_redis_get): + """Test error when private key file doesn't exist""" + mock_redis_get.return_value = None + mock_storage_load.side_effect = FileNotFoundError() + + with pytest.raises(PrivkeyNotFoundError) as exc_info: + get_decrypt_decoding("tenant-123") + + assert "Private key not found, tenant_id: tenant-123" in str(exc_info.value) + + +class TestEncryptDecryptIntegration: + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + @patch("libs.rsa.decrypt") + def test_should_encrypt_and_decrypt_consistently(self, mock_decrypt, mock_encrypt, mock_query): + """Test that encryption and decryption are consistent""" + # Setup mock tenant + mock_tenant = MagicMock() + mock_tenant.encrypt_public_key = "mock_public_key" + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + + # Setup mock encryption/decryption + original_token = "test_token_123" + mock_encrypt.return_value = b"encrypted_data" + mock_decrypt.return_value = original_token + + # Test encryption + encrypted = encrypt_token("tenant-123", original_token) + + # Test decryption + decrypted = decrypt_token("tenant-123", encrypted) + + assert decrypted == original_token + + +class TestSecurity: + """Critical security tests for encryption system""" + + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + def test_cross_tenant_isolation(self, mock_encrypt, mock_query): + """Ensure tokens encrypted for one tenant cannot be used by another""" + # Setup mock tenant + mock_tenant = MagicMock() + mock_tenant.encrypt_public_key = "tenant1_public_key" + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + mock_encrypt.return_value = b"encrypted_for_tenant1" + + # Encrypt token for tenant1 + encrypted = encrypt_token("tenant-123", "sensitive_data") + + # Attempt to decrypt with different tenant should fail + with patch("libs.rsa.decrypt") as mock_decrypt: + mock_decrypt.side_effect = Exception("Invalid tenant key") + + with pytest.raises(Exception, match="Invalid tenant key"): + decrypt_token("different-tenant", encrypted) + + @patch("libs.rsa.decrypt") + def test_tampered_ciphertext_rejection(self, mock_decrypt): + """Detect and reject tampered ciphertext""" + valid_encrypted = base64.b64encode(b"valid_data").decode() + + # Tamper with ciphertext + tampered_bytes = bytearray(base64.b64decode(valid_encrypted)) + tampered_bytes[0] ^= 0xFF + tampered = base64.b64encode(bytes(tampered_bytes)).decode() + + mock_decrypt.side_effect = Exception("Decryption error") + + with pytest.raises(Exception, match="Decryption error"): + decrypt_token("tenant-123", tampered) + + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + def test_encryption_randomness(self, mock_encrypt, mock_query): + """Ensure same plaintext produces different ciphertext""" + mock_tenant = MagicMock(encrypt_public_key="key") + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + + # Different outputs for same input + mock_encrypt.side_effect = [b"enc1", b"enc2", b"enc3"] + + results = [encrypt_token("tenant-123", "token") for _ in range(3)] + + # All results should be different + assert len(set(results)) == 3 + + +class TestEdgeCases: + """Additional security-focused edge case tests""" + + def test_should_handle_empty_string_in_obfuscation(self): + """Test handling of empty string in obfuscation""" + # Test empty string (which is a valid str type) + assert obfuscated_token("") == "" + + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + def test_should_handle_empty_token_encryption(self, mock_encrypt, mock_query): + """Test encryption of empty token""" + mock_tenant = MagicMock() + mock_tenant.encrypt_public_key = "mock_public_key" + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + mock_encrypt.return_value = b"encrypted_empty" + + result = encrypt_token("tenant-123", "") + + assert result == base64.b64encode(b"encrypted_empty").decode() + mock_encrypt.assert_called_with("", "mock_public_key") + + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + def test_should_handle_special_characters_in_token(self, mock_encrypt, mock_query): + """Test tokens containing special/unicode characters""" + mock_tenant = MagicMock() + mock_tenant.encrypt_public_key = "mock_public_key" + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + mock_encrypt.return_value = b"encrypted_special" + + # Test various special characters + special_tokens = [ + "token\x00with\x00null", # Null bytes + "token_with_emoji_😀🎉", # Unicode emoji + "token\nwith\nnewlines", # Newlines + "token\twith\ttabs", # Tabs + "token_with_中文字符", # Chinese characters + ] + + for token in special_tokens: + result = encrypt_token("tenant-123", token) + assert result == base64.b64encode(b"encrypted_special").decode() + mock_encrypt.assert_called_with(token, "mock_public_key") + + @patch("models.engine.db.session.query") + @patch("libs.rsa.encrypt") + def test_should_handle_rsa_size_limits(self, mock_encrypt, mock_query): + """Test behavior when token exceeds RSA encryption limits""" + mock_tenant = MagicMock() + mock_tenant.encrypt_public_key = "mock_public_key" + mock_query.return_value.filter.return_value.first.return_value = mock_tenant + + # RSA 2048-bit can only encrypt ~245 bytes + # The actual limit depends on padding scheme + mock_encrypt.side_effect = ValueError("Message too long for RSA key size") + + # Create a token that would exceed RSA limits + long_token = "x" * 300 + + with pytest.raises(ValueError, match="Message too long for RSA key size"): + encrypt_token("tenant-123", long_token) + + @patch("libs.rsa.get_decrypt_decoding") + @patch("libs.rsa.decrypt_token_with_decoding") + def test_batch_decrypt_loads_key_only_once(self, mock_decrypt_with_decoding, mock_get_decoding): + """Verify batch decryption optimization - loads key only once""" + mock_rsa_key = MagicMock() + mock_cipher_rsa = MagicMock() + mock_get_decoding.return_value = (mock_rsa_key, mock_cipher_rsa) + + # Test with multiple tokens + mock_decrypt_with_decoding.side_effect = ["token1", "token2", "token3", "token4", "token5"] + tokens = [base64.b64encode(f"encrypted{i}".encode()).decode() for i in range(5)] + + result = batch_decrypt_token("tenant-123", tokens) + + assert result == ["token1", "token2", "token3", "token4", "token5"] + # Key should only be loaded once regardless of token count + mock_get_decoding.assert_called_once_with("tenant-123") + assert mock_decrypt_with_decoding.call_count == 5 diff --git a/api/tests/unit_tests/core/mcp/client/test_session.py b/api/tests/unit_tests/core/mcp/client/test_session.py new file mode 100644 index 0000000000..c84169bf15 --- /dev/null +++ b/api/tests/unit_tests/core/mcp/client/test_session.py @@ -0,0 +1,471 @@ +import queue +import threading +from typing import Any + +from core.mcp import types +from core.mcp.entities import RequestContext +from core.mcp.session.base_session import RequestResponder +from core.mcp.session.client_session import DEFAULT_CLIENT_INFO, ClientSession +from core.mcp.types import ( + LATEST_PROTOCOL_VERSION, + ClientNotification, + ClientRequest, + Implementation, + InitializedNotification, + InitializeRequest, + InitializeResult, + JSONRPCMessage, + JSONRPCNotification, + JSONRPCRequest, + JSONRPCResponse, + ServerCapabilities, + ServerResult, + SessionMessage, +) + + +def test_client_session_initialize(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + initialized_notification = None + + def mock_server(): + nonlocal initialized_notification + + # Receive initialization request + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + + # Create response + result = ServerResult( + InitializeResult( + protocolVersion=LATEST_PROTOCOL_VERSION, + capabilities=ServerCapabilities( + logging=None, + resources=None, + tools=None, + experimental=None, + prompts=None, + ), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + instructions="The server instructions.", + ) + ) + + # Send response + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + + # Receive initialized notification + session_notification = client_to_server.get(timeout=5.0) + jsonrpc_notification = session_notification.message + assert isinstance(jsonrpc_notification.root, JSONRPCNotification) + initialized_notification = ClientNotification.model_validate( + jsonrpc_notification.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + + # Create message handler + def message_handler( + message: RequestResponder[types.ServerRequest, types.ClientResult] | types.ServerNotification | Exception, + ) -> None: + if isinstance(message, Exception): + raise message + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + # Create and use client session + with ClientSession( + server_to_client, + client_to_server, + message_handler=message_handler, + ) as session: + result = session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + # Assert results + assert isinstance(result, InitializeResult) + assert result.protocolVersion == LATEST_PROTOCOL_VERSION + assert isinstance(result.capabilities, ServerCapabilities) + assert result.serverInfo == Implementation(name="mock-server", version="0.1.0") + assert result.instructions == "The server instructions." + + # Check that client sent initialized notification + assert initialized_notification + assert isinstance(initialized_notification.root, InitializedNotification) + + +def test_client_session_custom_client_info(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + custom_client_info = Implementation(name="test-client", version="1.2.3") + received_client_info = None + + def mock_server(): + nonlocal received_client_info + + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + received_client_info = request.root.params.clientInfo + + result = ServerResult( + InitializeResult( + protocolVersion=LATEST_PROTOCOL_VERSION, + capabilities=ServerCapabilities(), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + ) + ) + + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + # Receive initialized notification + client_to_server.get(timeout=5.0) + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + with ClientSession( + server_to_client, + client_to_server, + client_info=custom_client_info, + ) as session: + session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + # Assert that custom client info was sent + assert received_client_info == custom_client_info + + +def test_client_session_default_client_info(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + received_client_info = None + + def mock_server(): + nonlocal received_client_info + + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + received_client_info = request.root.params.clientInfo + + result = ServerResult( + InitializeResult( + protocolVersion=LATEST_PROTOCOL_VERSION, + capabilities=ServerCapabilities(), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + ) + ) + + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + # Receive initialized notification + client_to_server.get(timeout=5.0) + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + with ClientSession( + server_to_client, + client_to_server, + ) as session: + session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + # Assert that default client info was used + assert received_client_info == DEFAULT_CLIENT_INFO + + +def test_client_session_version_negotiation_success(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + def mock_server(): + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + + # Send supported protocol version + result = ServerResult( + InitializeResult( + protocolVersion=LATEST_PROTOCOL_VERSION, + capabilities=ServerCapabilities(), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + ) + ) + + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + # Receive initialized notification + client_to_server.get(timeout=5.0) + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + with ClientSession( + server_to_client, + client_to_server, + ) as session: + result = session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + # Should successfully initialize + assert isinstance(result, InitializeResult) + assert result.protocolVersion == LATEST_PROTOCOL_VERSION + + +def test_client_session_version_negotiation_failure(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + def mock_server(): + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + + # Send unsupported protocol version + result = ServerResult( + InitializeResult( + protocolVersion="99.99.99", # Unsupported version + capabilities=ServerCapabilities(), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + ) + ) + + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + with ClientSession( + server_to_client, + client_to_server, + ) as session: + import pytest + + with pytest.raises(RuntimeError, match="Unsupported protocol version"): + session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + +def test_client_capabilities_default(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + received_capabilities = None + + def mock_server(): + nonlocal received_capabilities + + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + received_capabilities = request.root.params.capabilities + + result = ServerResult( + InitializeResult( + protocolVersion=LATEST_PROTOCOL_VERSION, + capabilities=ServerCapabilities(), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + ) + ) + + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + # Receive initialized notification + client_to_server.get(timeout=5.0) + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + with ClientSession( + server_to_client, + client_to_server, + ) as session: + session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + # Assert default capabilities + assert received_capabilities is not None + assert received_capabilities.sampling is not None + assert received_capabilities.roots is not None + assert received_capabilities.roots.listChanged is True + + +def test_client_capabilities_with_custom_callbacks(): + # Create synchronous queues to replace async streams + client_to_server: queue.Queue[SessionMessage] = queue.Queue() + server_to_client: queue.Queue[SessionMessage] = queue.Queue() + + def custom_sampling_callback( + context: RequestContext["ClientSession", Any], + params: types.CreateMessageRequestParams, + ) -> types.CreateMessageResult | types.ErrorData: + return types.CreateMessageResult( + model="test-model", + role="assistant", + content=types.TextContent(type="text", text="Custom response"), + ) + + def custom_list_roots_callback( + context: RequestContext["ClientSession", Any], + ) -> types.ListRootsResult | types.ErrorData: + return types.ListRootsResult(roots=[]) + + def mock_server(): + session_message = client_to_server.get(timeout=5.0) + jsonrpc_request = session_message.message + assert isinstance(jsonrpc_request.root, JSONRPCRequest) + request = ClientRequest.model_validate( + jsonrpc_request.root.model_dump(by_alias=True, mode="json", exclude_none=True) + ) + assert isinstance(request.root, InitializeRequest) + + result = ServerResult( + InitializeResult( + protocolVersion=LATEST_PROTOCOL_VERSION, + capabilities=ServerCapabilities(), + serverInfo=Implementation(name="mock-server", version="0.1.0"), + ) + ) + + server_to_client.put( + SessionMessage( + message=JSONRPCMessage( + JSONRPCResponse( + jsonrpc="2.0", + id=jsonrpc_request.root.id, + result=result.model_dump(by_alias=True, mode="json", exclude_none=True), + ) + ) + ) + ) + # Receive initialized notification + client_to_server.get(timeout=5.0) + + # Start mock server thread + server_thread = threading.Thread(target=mock_server, daemon=True) + server_thread.start() + + with ClientSession( + server_to_client, + client_to_server, + sampling_callback=custom_sampling_callback, + list_roots_callback=custom_list_roots_callback, + ) as session: + result = session.initialize() + + # Wait for server thread to complete + server_thread.join(timeout=10.0) + + # Verify initialization succeeded + assert isinstance(result, InitializeResult) + assert result.protocolVersion == LATEST_PROTOCOL_VERSION diff --git a/api/tests/unit_tests/core/mcp/client/test_sse.py b/api/tests/unit_tests/core/mcp/client/test_sse.py new file mode 100644 index 0000000000..8122cd08eb --- /dev/null +++ b/api/tests/unit_tests/core/mcp/client/test_sse.py @@ -0,0 +1,349 @@ +import json +import queue +import threading +import time +from typing import Any +from unittest.mock import Mock, patch + +import httpx +import pytest + +from core.mcp import types +from core.mcp.client.sse_client import sse_client +from core.mcp.error import MCPAuthError, MCPConnectionError + +SERVER_NAME = "test_server_for_SSE" + + +def test_sse_message_id_coercion(): + """Test that string message IDs that look like integers are parsed as integers. + + See for more details. + """ + json_message = '{"jsonrpc": "2.0", "id": "123", "method": "ping", "params": null}' + msg = types.JSONRPCMessage.model_validate_json(json_message) + expected = types.JSONRPCMessage(root=types.JSONRPCRequest(method="ping", jsonrpc="2.0", id=123)) + + # Check if both are JSONRPCRequest instances + assert isinstance(msg.root, types.JSONRPCRequest) + assert isinstance(expected.root, types.JSONRPCRequest) + + assert msg.root.id == expected.root.id + assert msg.root.method == expected.root.method + assert msg.root.jsonrpc == expected.root.jsonrpc + + +class MockSSEClient: + """Mock SSE client for testing.""" + + def __init__(self, url: str, headers: dict[str, Any] | None = None): + self.url = url + self.headers = headers or {} + self.connected = False + self.read_queue: queue.Queue = queue.Queue() + self.write_queue: queue.Queue = queue.Queue() + + def connect(self): + """Simulate connection establishment.""" + self.connected = True + + # Send endpoint event + endpoint_data = "/messages/?session_id=test-session-123" + self.read_queue.put(("endpoint", endpoint_data)) + + return self.read_queue, self.write_queue + + def send_initialize_response(self): + """Send a mock initialize response.""" + response = { + "jsonrpc": "2.0", + "id": 1, + "result": { + "protocolVersion": types.LATEST_PROTOCOL_VERSION, + "capabilities": { + "logging": None, + "resources": None, + "tools": None, + "experimental": None, + "prompts": None, + }, + "serverInfo": {"name": SERVER_NAME, "version": "0.1.0"}, + "instructions": "Test server instructions.", + }, + } + self.read_queue.put(("message", json.dumps(response))) + + +def test_sse_client_message_id_handling(): + """Test SSE client properly handles message ID coercion.""" + mock_client = MockSSEClient("http://test.example/sse") + read_queue, write_queue = mock_client.connect() + + # Send a message with string ID that should be coerced to int + message_data = { + "jsonrpc": "2.0", + "id": "456", # String ID + "result": {"test": "data"}, + } + read_queue.put(("message", json.dumps(message_data))) + read_queue.get(timeout=1.0) + # Get the message from queue + event_type, data = read_queue.get(timeout=1.0) + assert event_type == "message" + + # Parse the message + parsed_message = types.JSONRPCMessage.model_validate_json(data) + # Check that it's a JSONRPCResponse and verify the ID + assert isinstance(parsed_message.root, types.JSONRPCResponse) + assert parsed_message.root.id == 456 # Should be converted to int + + +def test_sse_client_connection_validation(): + """Test SSE client validates endpoint URLs properly.""" + test_url = "http://test.example/sse" + + with patch("core.mcp.client.sse_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + with patch("core.mcp.client.sse_client.ssrf_proxy_sse_connect") as mock_sse_connect: + # Mock the HTTP client + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + # Mock the SSE connection + mock_event_source = Mock() + mock_event_source.response.raise_for_status.return_value = None + mock_sse_connect.return_value.__enter__.return_value = mock_event_source + + # Mock SSE events + class MockSSEEvent: + def __init__(self, event_type: str, data: str): + self.event = event_type + self.data = data + + # Simulate endpoint event + endpoint_event = MockSSEEvent("endpoint", "/messages/?session_id=test-123") + mock_event_source.iter_sse.return_value = [endpoint_event] + + # Test connection + try: + with sse_client(test_url) as (read_queue, write_queue): + assert read_queue is not None + assert write_queue is not None + except Exception as e: + # Connection might fail due to mocking, but we're testing the validation logic + pass + + +def test_sse_client_error_handling(): + """Test SSE client properly handles various error conditions.""" + test_url = "http://test.example/sse" + + # Test 401 error handling + with patch("core.mcp.client.sse_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + with patch("core.mcp.client.sse_client.ssrf_proxy_sse_connect") as mock_sse_connect: + # Mock 401 HTTP error + mock_error = httpx.HTTPStatusError("Unauthorized", request=Mock(), response=Mock(status_code=401)) + mock_sse_connect.side_effect = mock_error + + with pytest.raises(MCPAuthError): + with sse_client(test_url): + pass + + # Test other HTTP errors + with patch("core.mcp.client.sse_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + with patch("core.mcp.client.sse_client.ssrf_proxy_sse_connect") as mock_sse_connect: + # Mock other HTTP error + mock_error = httpx.HTTPStatusError("Server Error", request=Mock(), response=Mock(status_code=500)) + mock_sse_connect.side_effect = mock_error + + with pytest.raises(MCPConnectionError): + with sse_client(test_url): + pass + + +def test_sse_client_timeout_configuration(): + """Test SSE client timeout configuration.""" + test_url = "http://test.example/sse" + custom_timeout = 10.0 + custom_sse_timeout = 300.0 + custom_headers = {"Authorization": "Bearer test-token"} + + with patch("core.mcp.client.sse_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + with patch("core.mcp.client.sse_client.ssrf_proxy_sse_connect") as mock_sse_connect: + # Mock successful connection + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + mock_event_source = Mock() + mock_event_source.response.raise_for_status.return_value = None + mock_event_source.iter_sse.return_value = [] + mock_sse_connect.return_value.__enter__.return_value = mock_event_source + + try: + with sse_client( + test_url, headers=custom_headers, timeout=custom_timeout, sse_read_timeout=custom_sse_timeout + ) as (read_queue, write_queue): + # Verify the configuration was passed correctly + mock_client_factory.assert_called_with(headers=custom_headers) + + # Check that timeout was configured + call_args = mock_sse_connect.call_args + assert call_args is not None + timeout_arg = call_args[1]["timeout"] + assert timeout_arg.read == custom_sse_timeout + except Exception: + # Connection might fail due to mocking, but we tested the configuration + pass + + +def test_sse_transport_endpoint_validation(): + """Test SSE transport validates endpoint URLs correctly.""" + from core.mcp.client.sse_client import SSETransport + + transport = SSETransport("http://example.com/sse") + + # Valid endpoint (same origin) + valid_endpoint = "http://example.com/messages/session123" + assert transport._validate_endpoint_url(valid_endpoint) == True + + # Invalid endpoint (different origin) + invalid_endpoint = "http://malicious.com/messages/session123" + assert transport._validate_endpoint_url(invalid_endpoint) == False + + # Invalid endpoint (different scheme) + invalid_scheme = "https://example.com/messages/session123" + assert transport._validate_endpoint_url(invalid_scheme) == False + + +def test_sse_transport_message_parsing(): + """Test SSE transport properly parses different message types.""" + from core.mcp.client.sse_client import SSETransport + + transport = SSETransport("http://example.com/sse") + read_queue: queue.Queue = queue.Queue() + + # Test valid JSON-RPC message + valid_message = '{"jsonrpc": "2.0", "id": 1, "method": "ping"}' + transport._handle_message_event(valid_message, read_queue) + + # Should have a SessionMessage in the queue + message = read_queue.get(timeout=1.0) + assert message is not None + assert hasattr(message, "message") + + # Test invalid JSON + invalid_json = '{"invalid": json}' + transport._handle_message_event(invalid_json, read_queue) + + # Should have an exception in the queue + error = read_queue.get(timeout=1.0) + assert isinstance(error, Exception) + + +def test_sse_client_queue_cleanup(): + """Test that SSE client properly cleans up queues on exit.""" + test_url = "http://test.example/sse" + + read_queue = None + write_queue = None + + with patch("core.mcp.client.sse_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + with patch("core.mcp.client.sse_client.ssrf_proxy_sse_connect") as mock_sse_connect: + # Mock connection that raises an exception + mock_sse_connect.side_effect = Exception("Connection failed") + + try: + with sse_client(test_url) as (rq, wq): + read_queue = rq + write_queue = wq + except Exception: + pass # Expected to fail + + # Queues should be cleaned up even on exception + # Note: In real implementation, cleanup should put None to signal shutdown + + +def test_sse_client_url_processing(): + """Test SSE client URL processing functions.""" + from core.mcp.client.sse_client import remove_request_params + + # Test URL with parameters + url_with_params = "http://example.com/sse?param1=value1¶m2=value2" + cleaned_url = remove_request_params(url_with_params) + assert cleaned_url == "http://example.com/sse" + + # Test URL without parameters + url_without_params = "http://example.com/sse" + cleaned_url = remove_request_params(url_without_params) + assert cleaned_url == "http://example.com/sse" + + # Test URL with path and parameters + complex_url = "http://example.com/path/to/sse?session=123&token=abc" + cleaned_url = remove_request_params(complex_url) + assert cleaned_url == "http://example.com/path/to/sse" + + +def test_sse_client_headers_propagation(): + """Test that custom headers are properly propagated in SSE client.""" + test_url = "http://test.example/sse" + custom_headers = { + "Authorization": "Bearer test-token", + "X-Custom-Header": "test-value", + "User-Agent": "test-client/1.0", + } + + with patch("core.mcp.client.sse_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + with patch("core.mcp.client.sse_client.ssrf_proxy_sse_connect") as mock_sse_connect: + # Mock the client factory to capture headers + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + # Mock the SSE connection + mock_event_source = Mock() + mock_event_source.response.raise_for_status.return_value = None + mock_event_source.iter_sse.return_value = [] + mock_sse_connect.return_value.__enter__.return_value = mock_event_source + + try: + with sse_client(test_url, headers=custom_headers): + pass + except Exception: + pass # Expected due to mocking + + # Verify headers were passed to client factory + mock_client_factory.assert_called_with(headers=custom_headers) + + +def test_sse_client_concurrent_access(): + """Test SSE client behavior with concurrent queue access.""" + test_read_queue: queue.Queue = queue.Queue() + + # Simulate concurrent producers and consumers + def producer(): + for i in range(10): + test_read_queue.put(f"message_{i}") + time.sleep(0.01) # Small delay to simulate real conditions + + def consumer(): + received = [] + for _ in range(10): + try: + msg = test_read_queue.get(timeout=2.0) + received.append(msg) + except queue.Empty: + break + return received + + # Start producer in separate thread + producer_thread = threading.Thread(target=producer, daemon=True) + producer_thread.start() + + # Consume messages + received_messages = consumer() + + # Wait for producer to finish + producer_thread.join(timeout=5.0) + + # Verify all messages were received + assert len(received_messages) == 10 + for i in range(10): + assert f"message_{i}" in received_messages diff --git a/api/tests/unit_tests/core/mcp/client/test_streamable_http.py b/api/tests/unit_tests/core/mcp/client/test_streamable_http.py new file mode 100644 index 0000000000..9a30a35a49 --- /dev/null +++ b/api/tests/unit_tests/core/mcp/client/test_streamable_http.py @@ -0,0 +1,450 @@ +""" +Tests for the StreamableHTTP client transport. + +Contains tests for only the client side of the StreamableHTTP transport. +""" + +import queue +import threading +import time +from typing import Any +from unittest.mock import Mock, patch + +from core.mcp import types +from core.mcp.client.streamable_client import streamablehttp_client + +# Test constants +SERVER_NAME = "test_streamable_http_server" +TEST_SESSION_ID = "test-session-id-12345" +INIT_REQUEST = { + "jsonrpc": "2.0", + "method": "initialize", + "params": { + "clientInfo": {"name": "test-client", "version": "1.0"}, + "protocolVersion": "2025-03-26", + "capabilities": {}, + }, + "id": "init-1", +} + + +class MockStreamableHTTPClient: + """Mock StreamableHTTP client for testing.""" + + def __init__(self, url: str, headers: dict[str, Any] | None = None): + self.url = url + self.headers = headers or {} + self.connected = False + self.read_queue: queue.Queue = queue.Queue() + self.write_queue: queue.Queue = queue.Queue() + self.session_id = TEST_SESSION_ID + + def connect(self): + """Simulate connection establishment.""" + self.connected = True + return self.read_queue, self.write_queue, lambda: self.session_id + + def send_initialize_response(self): + """Send a mock initialize response.""" + session_message = types.SessionMessage( + message=types.JSONRPCMessage( + root=types.JSONRPCResponse( + jsonrpc="2.0", + id="init-1", + result={ + "protocolVersion": types.LATEST_PROTOCOL_VERSION, + "capabilities": { + "logging": None, + "resources": None, + "tools": None, + "experimental": None, + "prompts": None, + }, + "serverInfo": {"name": SERVER_NAME, "version": "0.1.0"}, + "instructions": "Test server instructions.", + }, + ) + ) + ) + self.read_queue.put(session_message) + + def send_tools_response(self): + """Send a mock tools list response.""" + session_message = types.SessionMessage( + message=types.JSONRPCMessage( + root=types.JSONRPCResponse( + jsonrpc="2.0", + id="tools-1", + result={ + "tools": [ + { + "name": "test_tool", + "description": "A test tool", + "inputSchema": {"type": "object", "properties": {}}, + } + ], + }, + ) + ) + ) + self.read_queue.put(session_message) + + +def test_streamablehttp_client_message_id_handling(): + """Test StreamableHTTP client properly handles message ID coercion.""" + mock_client = MockStreamableHTTPClient("http://test.example/mcp") + read_queue, write_queue, get_session_id = mock_client.connect() + + # Send a message with string ID that should be coerced to int + response_message = types.SessionMessage( + message=types.JSONRPCMessage(root=types.JSONRPCResponse(jsonrpc="2.0", id="789", result={"test": "data"})) + ) + read_queue.put(response_message) + + # Get the message from queue + message = read_queue.get(timeout=1.0) + assert message is not None + assert isinstance(message, types.SessionMessage) + + # Check that the ID was properly handled + assert isinstance(message.message.root, types.JSONRPCResponse) + assert message.message.root.id == 789 # ID should be coerced to int due to union_mode="left_to_right" + + +def test_streamablehttp_client_connection_validation(): + """Test StreamableHTTP client validates connections properly.""" + test_url = "http://test.example/mcp" + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + # Mock the HTTP client + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + # Mock successful response + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.raise_for_status.return_value = None + mock_client.post.return_value = mock_response + + # Test connection + try: + with streamablehttp_client(test_url) as (read_queue, write_queue, get_session_id): + assert read_queue is not None + assert write_queue is not None + assert get_session_id is not None + except Exception: + # Connection might fail due to mocking, but we're testing the validation logic + pass + + +def test_streamablehttp_client_timeout_configuration(): + """Test StreamableHTTP client timeout configuration.""" + test_url = "http://test.example/mcp" + custom_headers = {"Authorization": "Bearer test-token"} + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + # Mock successful connection + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.raise_for_status.return_value = None + mock_client.post.return_value = mock_response + + try: + with streamablehttp_client(test_url, headers=custom_headers) as (read_queue, write_queue, get_session_id): + # Verify the configuration was passed correctly + mock_client_factory.assert_called_with(headers=custom_headers) + except Exception: + # Connection might fail due to mocking, but we tested the configuration + pass + + +def test_streamablehttp_client_session_id_handling(): + """Test StreamableHTTP client properly handles session IDs.""" + mock_client = MockStreamableHTTPClient("http://test.example/mcp") + read_queue, write_queue, get_session_id = mock_client.connect() + + # Test that session ID is available + session_id = get_session_id() + assert session_id == TEST_SESSION_ID + + # Test that we can use the session ID in subsequent requests + assert session_id is not None + assert len(session_id) > 0 + + +def test_streamablehttp_client_message_parsing(): + """Test StreamableHTTP client properly parses different message types.""" + mock_client = MockStreamableHTTPClient("http://test.example/mcp") + read_queue, write_queue, get_session_id = mock_client.connect() + + # Test valid initialization response + mock_client.send_initialize_response() + + # Should have a SessionMessage in the queue + message = read_queue.get(timeout=1.0) + assert message is not None + assert isinstance(message, types.SessionMessage) + assert isinstance(message.message.root, types.JSONRPCResponse) + + # Test tools response + mock_client.send_tools_response() + + tools_message = read_queue.get(timeout=1.0) + assert tools_message is not None + assert isinstance(tools_message, types.SessionMessage) + + +def test_streamablehttp_client_queue_cleanup(): + """Test that StreamableHTTP client properly cleans up queues on exit.""" + test_url = "http://test.example/mcp" + + read_queue = None + write_queue = None + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + # Mock connection that raises an exception + mock_client_factory.side_effect = Exception("Connection failed") + + try: + with streamablehttp_client(test_url) as (rq, wq, get_session_id): + read_queue = rq + write_queue = wq + except Exception: + pass # Expected to fail + + # Queues should be cleaned up even on exception + # Note: In real implementation, cleanup should put None to signal shutdown + + +def test_streamablehttp_client_headers_propagation(): + """Test that custom headers are properly propagated in StreamableHTTP client.""" + test_url = "http://test.example/mcp" + custom_headers = { + "Authorization": "Bearer test-token", + "X-Custom-Header": "test-value", + "User-Agent": "test-client/1.0", + } + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + # Mock the client factory to capture headers + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.raise_for_status.return_value = None + mock_client.post.return_value = mock_response + + try: + with streamablehttp_client(test_url, headers=custom_headers): + pass + except Exception: + pass # Expected due to mocking + + # Verify headers were passed to client factory + # Check that the call was made with headers that include our custom headers + mock_client_factory.assert_called_once() + call_args = mock_client_factory.call_args + assert "headers" in call_args.kwargs + passed_headers = call_args.kwargs["headers"] + + # Verify all custom headers are present + for key, value in custom_headers.items(): + assert key in passed_headers + assert passed_headers[key] == value + + +def test_streamablehttp_client_concurrent_access(): + """Test StreamableHTTP client behavior with concurrent queue access.""" + test_read_queue: queue.Queue = queue.Queue() + test_write_queue: queue.Queue = queue.Queue() + + # Simulate concurrent producers and consumers + def producer(): + for i in range(10): + test_read_queue.put(f"message_{i}") + time.sleep(0.01) # Small delay to simulate real conditions + + def consumer(): + received = [] + for _ in range(10): + try: + msg = test_read_queue.get(timeout=2.0) + received.append(msg) + except queue.Empty: + break + return received + + # Start producer in separate thread + producer_thread = threading.Thread(target=producer, daemon=True) + producer_thread.start() + + # Consume messages + received_messages = consumer() + + # Wait for producer to finish + producer_thread.join(timeout=5.0) + + # Verify all messages were received + assert len(received_messages) == 10 + for i in range(10): + assert f"message_{i}" in received_messages + + +def test_streamablehttp_client_json_vs_sse_mode(): + """Test StreamableHTTP client handling of JSON vs SSE response modes.""" + test_url = "http://test.example/mcp" + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + # Mock JSON response + mock_json_response = Mock() + mock_json_response.status_code = 200 + mock_json_response.headers = {"content-type": "application/json"} + mock_json_response.json.return_value = {"result": "json_mode"} + mock_json_response.raise_for_status.return_value = None + + # Mock SSE response + mock_sse_response = Mock() + mock_sse_response.status_code = 200 + mock_sse_response.headers = {"content-type": "text/event-stream"} + mock_sse_response.raise_for_status.return_value = None + + # Test JSON mode + mock_client.post.return_value = mock_json_response + + try: + with streamablehttp_client(test_url) as (read_queue, write_queue, get_session_id): + # Should handle JSON responses + assert read_queue is not None + assert write_queue is not None + except Exception: + pass # Expected due to mocking + + # Test SSE mode + mock_client.post.return_value = mock_sse_response + + try: + with streamablehttp_client(test_url) as (read_queue, write_queue, get_session_id): + # Should handle SSE responses + assert read_queue is not None + assert write_queue is not None + except Exception: + pass # Expected due to mocking + + +def test_streamablehttp_client_terminate_on_close(): + """Test StreamableHTTP client terminate_on_close parameter.""" + test_url = "http://test.example/mcp" + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.raise_for_status.return_value = None + mock_client.post.return_value = mock_response + mock_client.delete.return_value = mock_response + + # Test with terminate_on_close=True (default) + try: + with streamablehttp_client(test_url, terminate_on_close=True) as (read_queue, write_queue, get_session_id): + pass + except Exception: + pass # Expected due to mocking + + # Test with terminate_on_close=False + try: + with streamablehttp_client(test_url, terminate_on_close=False) as (read_queue, write_queue, get_session_id): + pass + except Exception: + pass # Expected due to mocking + + +def test_streamablehttp_client_protocol_version_handling(): + """Test StreamableHTTP client protocol version handling.""" + mock_client = MockStreamableHTTPClient("http://test.example/mcp") + read_queue, write_queue, get_session_id = mock_client.connect() + + # Send initialize response with specific protocol version + + session_message = types.SessionMessage( + message=types.JSONRPCMessage( + root=types.JSONRPCResponse( + jsonrpc="2.0", + id="init-1", + result={ + "protocolVersion": "2024-11-05", + "capabilities": {}, + "serverInfo": {"name": SERVER_NAME, "version": "0.1.0"}, + }, + ) + ) + ) + read_queue.put(session_message) + + # Get the message and verify protocol version + message = read_queue.get(timeout=1.0) + assert message is not None + assert isinstance(message.message.root, types.JSONRPCResponse) + result = message.message.root.result + assert result["protocolVersion"] == "2024-11-05" + + +def test_streamablehttp_client_error_response_handling(): + """Test StreamableHTTP client handling of error responses.""" + mock_client = MockStreamableHTTPClient("http://test.example/mcp") + read_queue, write_queue, get_session_id = mock_client.connect() + + # Send an error response + session_message = types.SessionMessage( + message=types.JSONRPCMessage( + root=types.JSONRPCError( + jsonrpc="2.0", + id="test-1", + error=types.ErrorData(code=-32601, message="Method not found", data=None), + ) + ) + ) + read_queue.put(session_message) + + # Get the error message + message = read_queue.get(timeout=1.0) + assert message is not None + assert isinstance(message.message.root, types.JSONRPCError) + assert message.message.root.error.code == -32601 + assert message.message.root.error.message == "Method not found" + + +def test_streamablehttp_client_resumption_token_handling(): + """Test StreamableHTTP client resumption token functionality.""" + test_url = "http://test.example/mcp" + test_resumption_token = "resume-token-123" + + with patch("core.mcp.client.streamable_client.create_ssrf_proxy_mcp_http_client") as mock_client_factory: + mock_client = Mock() + mock_client_factory.return_value.__enter__.return_value = mock_client + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json", "last-event-id": test_resumption_token} + mock_response.raise_for_status.return_value = None + mock_client.post.return_value = mock_response + + try: + with streamablehttp_client(test_url) as (read_queue, write_queue, get_session_id): + # Test that resumption token can be captured from headers + assert read_queue is not None + assert write_queue is not None + except Exception: + pass # Expected due to mocking diff --git a/api/tests/unit_tests/core/repositories/__init__.py b/api/tests/unit_tests/core/repositories/__init__.py new file mode 100644 index 0000000000..c65d7da61d --- /dev/null +++ b/api/tests/unit_tests/core/repositories/__init__.py @@ -0,0 +1 @@ +# Unit tests for core repositories module diff --git a/api/tests/unit_tests/core/repositories/test_factory.py b/api/tests/unit_tests/core/repositories/test_factory.py new file mode 100644 index 0000000000..fce4a6fb6b --- /dev/null +++ b/api/tests/unit_tests/core/repositories/test_factory.py @@ -0,0 +1,455 @@ +""" +Unit tests for the RepositoryFactory. + +This module tests the factory pattern implementation for creating repository instances +based on configuration, including error handling and validation. +""" + +from unittest.mock import MagicMock, patch + +import pytest +from pytest_mock import MockerFixture +from sqlalchemy.engine import Engine +from sqlalchemy.orm import sessionmaker + +from core.repositories.factory import DifyCoreRepositoryFactory, RepositoryImportError +from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository +from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository +from models import Account, EndUser +from models.enums import WorkflowRunTriggeredFrom +from models.workflow import WorkflowNodeExecutionTriggeredFrom + + +class TestRepositoryFactory: + """Test cases for RepositoryFactory.""" + + def test_import_class_success(self): + """Test successful class import.""" + # Test importing a real class + class_path = "unittest.mock.MagicMock" + result = DifyCoreRepositoryFactory._import_class(class_path) + assert result is MagicMock + + def test_import_class_invalid_path(self): + """Test import with invalid module path.""" + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._import_class("invalid.module.path") + assert "Cannot import repository class" in str(exc_info.value) + + def test_import_class_invalid_class_name(self): + """Test import with invalid class name.""" + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._import_class("unittest.mock.NonExistentClass") + assert "Cannot import repository class" in str(exc_info.value) + + def test_import_class_malformed_path(self): + """Test import with malformed path (no dots).""" + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._import_class("invalidpath") + assert "Cannot import repository class" in str(exc_info.value) + + def test_validate_repository_interface_success(self): + """Test successful interface validation.""" + + # Create a mock class that implements the required methods + class MockRepository: + def save(self): + pass + + def get_by_id(self): + pass + + # Create a mock interface with the same methods + class MockInterface: + def save(self): + pass + + def get_by_id(self): + pass + + # Should not raise an exception + DifyCoreRepositoryFactory._validate_repository_interface(MockRepository, MockInterface) + + def test_validate_repository_interface_missing_methods(self): + """Test interface validation with missing methods.""" + + # Create a mock class that doesn't implement all required methods + class IncompleteRepository: + def save(self): + pass + + # Missing get_by_id method + + # Create a mock interface with required methods + class MockInterface: + def save(self): + pass + + def get_by_id(self): + pass + + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._validate_repository_interface(IncompleteRepository, MockInterface) + assert "does not implement required methods" in str(exc_info.value) + assert "get_by_id" in str(exc_info.value) + + def test_validate_constructor_signature_success(self): + """Test successful constructor signature validation.""" + + class MockRepository: + def __init__(self, session_factory, user, app_id, triggered_from): + pass + + # Should not raise an exception + DifyCoreRepositoryFactory._validate_constructor_signature( + MockRepository, ["session_factory", "user", "app_id", "triggered_from"] + ) + + def test_validate_constructor_signature_missing_params(self): + """Test constructor validation with missing parameters.""" + + class IncompleteRepository: + def __init__(self, session_factory, user): + # Missing app_id and triggered_from parameters + pass + + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._validate_constructor_signature( + IncompleteRepository, ["session_factory", "user", "app_id", "triggered_from"] + ) + assert "does not accept required parameters" in str(exc_info.value) + assert "app_id" in str(exc_info.value) + assert "triggered_from" in str(exc_info.value) + + def test_validate_constructor_signature_inspection_error(self, mocker: MockerFixture): + """Test constructor validation when inspection fails.""" + # Mock inspect.signature to raise an exception + mocker.patch("inspect.signature", side_effect=Exception("Inspection failed")) + + class MockRepository: + def __init__(self, session_factory): + pass + + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._validate_constructor_signature(MockRepository, ["session_factory"]) + assert "Failed to validate constructor signature" in str(exc_info.value) + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_execution_repository_success(self, mock_config, mocker: MockerFixture): + """Test successful creation of WorkflowExecutionRepository.""" + # Setup mock configuration + mock_config.WORKFLOW_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + # Create mock dependencies + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=Account) + app_id = "test-app-id" + triggered_from = WorkflowRunTriggeredFrom.APP_RUN + + # Mock the imported class to be a valid repository + mock_repository_class = MagicMock() + mock_repository_instance = MagicMock(spec=WorkflowExecutionRepository) + mock_repository_class.return_value = mock_repository_instance + + # Mock the validation methods + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object(DifyCoreRepositoryFactory, "_validate_repository_interface"), + patch.object(DifyCoreRepositoryFactory, "_validate_constructor_signature"), + ): + result = DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id=app_id, + triggered_from=triggered_from, + ) + + # Verify the repository was created with correct parameters + mock_repository_class.assert_called_once_with( + session_factory=mock_session_factory, + user=mock_user, + app_id=app_id, + triggered_from=triggered_from, + ) + assert result is mock_repository_instance + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_execution_repository_import_error(self, mock_config): + """Test WorkflowExecutionRepository creation with import error.""" + # Setup mock configuration with invalid class path + mock_config.WORKFLOW_EXECUTION_REPOSITORY = "invalid.module.InvalidClass" + + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=Account) + + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowRunTriggeredFrom.APP_RUN, + ) + assert "Cannot import repository class" in str(exc_info.value) + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_execution_repository_validation_error(self, mock_config, mocker: MockerFixture): + """Test WorkflowExecutionRepository creation with validation error.""" + # Setup mock configuration + mock_config.WORKFLOW_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=Account) + + # Mock import to succeed but validation to fail + mock_repository_class = MagicMock() + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object( + DifyCoreRepositoryFactory, + "_validate_repository_interface", + side_effect=RepositoryImportError("Interface validation failed"), + ), + ): + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowRunTriggeredFrom.APP_RUN, + ) + assert "Interface validation failed" in str(exc_info.value) + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_execution_repository_instantiation_error(self, mock_config, mocker: MockerFixture): + """Test WorkflowExecutionRepository creation with instantiation error.""" + # Setup mock configuration + mock_config.WORKFLOW_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=Account) + + # Mock import and validation to succeed but instantiation to fail + mock_repository_class = MagicMock(side_effect=Exception("Instantiation failed")) + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object(DifyCoreRepositoryFactory, "_validate_repository_interface"), + patch.object(DifyCoreRepositoryFactory, "_validate_constructor_signature"), + ): + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowRunTriggeredFrom.APP_RUN, + ) + assert "Failed to create WorkflowExecutionRepository" in str(exc_info.value) + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_node_execution_repository_success(self, mock_config, mocker: MockerFixture): + """Test successful creation of WorkflowNodeExecutionRepository.""" + # Setup mock configuration + mock_config.WORKFLOW_NODE_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + # Create mock dependencies + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=EndUser) + app_id = "test-app-id" + triggered_from = WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN + + # Mock the imported class to be a valid repository + mock_repository_class = MagicMock() + mock_repository_instance = MagicMock(spec=WorkflowNodeExecutionRepository) + mock_repository_class.return_value = mock_repository_instance + + # Mock the validation methods + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object(DifyCoreRepositoryFactory, "_validate_repository_interface"), + patch.object(DifyCoreRepositoryFactory, "_validate_constructor_signature"), + ): + result = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id=app_id, + triggered_from=triggered_from, + ) + + # Verify the repository was created with correct parameters + mock_repository_class.assert_called_once_with( + session_factory=mock_session_factory, + user=mock_user, + app_id=app_id, + triggered_from=triggered_from, + ) + assert result is mock_repository_instance + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_node_execution_repository_import_error(self, mock_config): + """Test WorkflowNodeExecutionRepository creation with import error.""" + # Setup mock configuration with invalid class path + mock_config.WORKFLOW_NODE_EXECUTION_REPOSITORY = "invalid.module.InvalidClass" + + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=EndUser) + + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory.create_workflow_node_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, + ) + assert "Cannot import repository class" in str(exc_info.value) + + def test_repository_import_error_exception(self): + """Test RepositoryImportError exception.""" + error_message = "Test error message" + exception = RepositoryImportError(error_message) + assert str(exception) == error_message + assert isinstance(exception, Exception) + + @patch("core.repositories.factory.dify_config") + def test_create_with_engine_instead_of_sessionmaker(self, mock_config, mocker: MockerFixture): + """Test repository creation with Engine instead of sessionmaker.""" + # Setup mock configuration + mock_config.WORKFLOW_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + # Create mock dependencies with Engine instead of sessionmaker + mock_engine = MagicMock(spec=Engine) + mock_user = MagicMock(spec=Account) + + # Mock the imported class to be a valid repository + mock_repository_class = MagicMock() + mock_repository_instance = MagicMock(spec=WorkflowExecutionRepository) + mock_repository_class.return_value = mock_repository_instance + + # Mock the validation methods + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object(DifyCoreRepositoryFactory, "_validate_repository_interface"), + patch.object(DifyCoreRepositoryFactory, "_validate_constructor_signature"), + ): + result = DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=mock_engine, # Using Engine instead of sessionmaker + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowRunTriggeredFrom.APP_RUN, + ) + + # Verify the repository was created with the Engine + mock_repository_class.assert_called_once_with( + session_factory=mock_engine, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowRunTriggeredFrom.APP_RUN, + ) + assert result is mock_repository_instance + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_node_execution_repository_validation_error(self, mock_config): + """Test WorkflowNodeExecutionRepository creation with validation error.""" + # Setup mock configuration + mock_config.WORKFLOW_NODE_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=EndUser) + + # Mock import to succeed but validation to fail + mock_repository_class = MagicMock() + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object( + DifyCoreRepositoryFactory, + "_validate_repository_interface", + side_effect=RepositoryImportError("Interface validation failed"), + ), + ): + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory.create_workflow_node_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, + ) + assert "Interface validation failed" in str(exc_info.value) + + @patch("core.repositories.factory.dify_config") + def test_create_workflow_node_execution_repository_instantiation_error(self, mock_config): + """Test WorkflowNodeExecutionRepository creation with instantiation error.""" + # Setup mock configuration + mock_config.WORKFLOW_NODE_EXECUTION_REPOSITORY = "unittest.mock.MagicMock" + + mock_session_factory = MagicMock(spec=sessionmaker) + mock_user = MagicMock(spec=EndUser) + + # Mock import and validation to succeed but instantiation to fail + mock_repository_class = MagicMock(side_effect=Exception("Instantiation failed")) + with ( + patch.object(DifyCoreRepositoryFactory, "_import_class", return_value=mock_repository_class), + patch.object(DifyCoreRepositoryFactory, "_validate_repository_interface"), + patch.object(DifyCoreRepositoryFactory, "_validate_constructor_signature"), + ): + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory.create_workflow_node_execution_repository( + session_factory=mock_session_factory, + user=mock_user, + app_id="test-app-id", + triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, + ) + assert "Failed to create WorkflowNodeExecutionRepository" in str(exc_info.value) + + def test_validate_repository_interface_with_private_methods(self): + """Test interface validation ignores private methods.""" + + # Create a mock class with private methods + class MockRepository: + def save(self): + pass + + def get_by_id(self): + pass + + def _private_method(self): + pass + + # Create a mock interface with private methods + class MockInterface: + def save(self): + pass + + def get_by_id(self): + pass + + def _private_method(self): + pass + + # Should not raise an exception (private methods are ignored) + DifyCoreRepositoryFactory._validate_repository_interface(MockRepository, MockInterface) + + def test_validate_constructor_signature_with_extra_params(self): + """Test constructor validation with extra parameters (should pass).""" + + class MockRepository: + def __init__(self, session_factory, user, app_id, triggered_from, extra_param=None): + pass + + # Should not raise an exception (extra parameters are allowed) + DifyCoreRepositoryFactory._validate_constructor_signature( + MockRepository, ["session_factory", "user", "app_id", "triggered_from"] + ) + + def test_validate_constructor_signature_with_kwargs(self): + """Test constructor validation with **kwargs (current implementation doesn't support this).""" + + class MockRepository: + def __init__(self, session_factory, user, **kwargs): + pass + + # Current implementation doesn't handle **kwargs, so this should raise an exception + with pytest.raises(RepositoryImportError) as exc_info: + DifyCoreRepositoryFactory._validate_constructor_signature( + MockRepository, ["session_factory", "user", "app_id", "triggered_from"] + ) + assert "does not accept required parameters" in str(exc_info.value) + assert "app_id" in str(exc_info.value) + assert "triggered_from" in str(exc_info.value) diff --git a/api/tests/unit_tests/core/tools/utils/__init__.py b/api/tests/unit_tests/core/tools/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/tools/utils/test_parser.py b/api/tests/unit_tests/core/tools/utils/test_parser.py new file mode 100644 index 0000000000..8e07293ce0 --- /dev/null +++ b/api/tests/unit_tests/core/tools/utils/test_parser.py @@ -0,0 +1,56 @@ +import pytest +from flask import Flask + +from core.tools.utils.parser import ApiBasedToolSchemaParser + + +@pytest.fixture +def app(): + app = Flask(__name__) + return app + + +def test_parse_openapi_to_tool_bundle_operation_id(app): + openapi = { + "openapi": "3.0.0", + "info": {"title": "Simple API", "version": "1.0.0"}, + "servers": [{"url": "http://localhost:3000"}], + "paths": { + "/": { + "get": { + "summary": "Root endpoint", + "responses": { + "200": { + "description": "Successful response", + } + }, + } + }, + "/api/resources": { + "get": { + "summary": "Non-root endpoint without an operationId", + "responses": { + "200": { + "description": "Successful response", + } + }, + }, + "post": { + "summary": "Non-root endpoint with an operationId", + "operationId": "createResource", + "responses": { + "201": { + "description": "Resource created", + } + }, + }, + }, + }, + } + with app.test_request_context(): + tool_bundles = ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi) + + assert len(tool_bundles) == 3 + assert tool_bundles[0].operation_id == "_get" + assert tool_bundles[1].operation_id == "apiresources_get" + assert tool_bundles[2].operation_id == "createResource" diff --git a/api/tests/unit_tests/extensions/test_redis.py b/api/tests/unit_tests/extensions/test_redis.py new file mode 100644 index 0000000000..933fa32894 --- /dev/null +++ b/api/tests/unit_tests/extensions/test_redis.py @@ -0,0 +1,53 @@ +from redis import RedisError + +from extensions.ext_redis import redis_fallback + + +def test_redis_fallback_success(): + @redis_fallback(default_return=None) + def test_func(): + return "success" + + assert test_func() == "success" + + +def test_redis_fallback_error(): + @redis_fallback(default_return="fallback") + def test_func(): + raise RedisError("Redis error") + + assert test_func() == "fallback" + + +def test_redis_fallback_none_default(): + @redis_fallback() + def test_func(): + raise RedisError("Redis error") + + assert test_func() is None + + +def test_redis_fallback_with_args(): + @redis_fallback(default_return=0) + def test_func(x, y): + raise RedisError("Redis error") + + assert test_func(1, 2) == 0 + + +def test_redis_fallback_with_kwargs(): + @redis_fallback(default_return={}) + def test_func(x=None, y=None): + raise RedisError("Redis error") + + assert test_func(x=1, y=2) == {} + + +def test_redis_fallback_preserves_function_metadata(): + @redis_fallback(default_return=None) + def test_func(): + """Test function docstring""" + pass + + assert test_func.__name__ == "test_func" + assert test_func.__doc__ == "Test function docstring" diff --git a/api/tests/unit_tests/factories/test_variable_factory.py b/api/tests/unit_tests/factories/test_variable_factory.py index 481fbdc91a..edd4c5e93e 100644 --- a/api/tests/unit_tests/factories/test_variable_factory.py +++ b/api/tests/unit_tests/factories/test_variable_factory.py @@ -14,9 +14,7 @@ from core.variables import ( ArrayStringVariable, FloatVariable, IntegerVariable, - ObjectSegment, SecretVariable, - SegmentType, StringVariable, ) from core.variables.exc import VariableError @@ -418,8 +416,6 @@ def test_build_segment_file_array_with_different_file_types(): @st.composite def _generate_file(draw) -> File: - file_id = draw(st.text(min_size=1, max_size=10)) - tenant_id = draw(st.text(min_size=1, max_size=10)) file_type, mime_type, extension = draw( st.sampled_from( [ diff --git a/api/tests/unit_tests/libs/test_helper.py b/api/tests/unit_tests/libs/test_helper.py new file mode 100644 index 0000000000..b7701055f5 --- /dev/null +++ b/api/tests/unit_tests/libs/test_helper.py @@ -0,0 +1,65 @@ +import pytest + +from libs.helper import extract_tenant_id +from models.account import Account +from models.model import EndUser + + +class TestExtractTenantId: + """Test cases for the extract_tenant_id utility function.""" + + def test_extract_tenant_id_from_account_with_tenant(self): + """Test extracting tenant_id from Account with current_tenant_id.""" + # Create a mock Account object + account = Account() + # Mock the current_tenant_id property + account._current_tenant = type("MockTenant", (), {"id": "account-tenant-123"})() + + tenant_id = extract_tenant_id(account) + assert tenant_id == "account-tenant-123" + + def test_extract_tenant_id_from_account_without_tenant(self): + """Test extracting tenant_id from Account without current_tenant_id.""" + # Create a mock Account object + account = Account() + account._current_tenant = None + + tenant_id = extract_tenant_id(account) + assert tenant_id is None + + def test_extract_tenant_id_from_enduser_with_tenant(self): + """Test extracting tenant_id from EndUser with tenant_id.""" + # Create a mock EndUser object + end_user = EndUser() + end_user.tenant_id = "enduser-tenant-456" + + tenant_id = extract_tenant_id(end_user) + assert tenant_id == "enduser-tenant-456" + + def test_extract_tenant_id_from_enduser_without_tenant(self): + """Test extracting tenant_id from EndUser without tenant_id.""" + # Create a mock EndUser object + end_user = EndUser() + end_user.tenant_id = None + + tenant_id = extract_tenant_id(end_user) + assert tenant_id is None + + def test_extract_tenant_id_with_invalid_user_type(self): + """Test extracting tenant_id with invalid user type raises ValueError.""" + invalid_user = "not_a_user_object" + + with pytest.raises(ValueError, match="Invalid user type.*Expected Account or EndUser"): + extract_tenant_id(invalid_user) + + def test_extract_tenant_id_with_none_user(self): + """Test extracting tenant_id with None user raises ValueError.""" + with pytest.raises(ValueError, match="Invalid user type.*Expected Account or EndUser"): + extract_tenant_id(None) + + def test_extract_tenant_id_with_dict_user(self): + """Test extracting tenant_id with dict user raises ValueError.""" + dict_user = {"id": "123", "tenant_id": "456"} + + with pytest.raises(ValueError, match="Invalid user type.*Expected Account or EndUser"): + extract_tenant_id(dict_user) diff --git a/api/tests/unit_tests/libs/test_login.py b/api/tests/unit_tests/libs/test_login.py new file mode 100644 index 0000000000..39671077d4 --- /dev/null +++ b/api/tests/unit_tests/libs/test_login.py @@ -0,0 +1,232 @@ +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask, g +from flask_login import LoginManager, UserMixin + +from libs.login import _get_user, current_user, login_required + + +class MockUser(UserMixin): + """Mock user class for testing.""" + + def __init__(self, id: str, is_authenticated: bool = True): + self.id = id + self._is_authenticated = is_authenticated + + @property + def is_authenticated(self): + return self._is_authenticated + + +class TestLoginRequired: + """Test cases for login_required decorator.""" + + @pytest.fixture + def setup_app(self, app: Flask): + """Set up Flask app with login manager.""" + # Initialize login manager + login_manager = LoginManager() + login_manager.init_app(app) + + # Mock unauthorized handler + login_manager.unauthorized = MagicMock(return_value="Unauthorized") + + # Add a dummy user loader to prevent exceptions + @login_manager.user_loader + def load_user(user_id): + return None + + return app + + def test_authenticated_user_can_access_protected_view(self, setup_app: Flask): + """Test that authenticated users can access protected views.""" + + @login_required + def protected_view(): + return "Protected content" + + with setup_app.test_request_context(): + # Mock authenticated user + mock_user = MockUser("test_user", is_authenticated=True) + with patch("libs.login._get_user", return_value=mock_user): + result = protected_view() + assert result == "Protected content" + + def test_unauthenticated_user_cannot_access_protected_view(self, setup_app: Flask): + """Test that unauthenticated users are redirected.""" + + @login_required + def protected_view(): + return "Protected content" + + with setup_app.test_request_context(): + # Mock unauthenticated user + mock_user = MockUser("test_user", is_authenticated=False) + with patch("libs.login._get_user", return_value=mock_user): + result = protected_view() + assert result == "Unauthorized" + setup_app.login_manager.unauthorized.assert_called_once() + + def test_login_disabled_allows_unauthenticated_access(self, setup_app: Flask): + """Test that LOGIN_DISABLED config bypasses authentication.""" + + @login_required + def protected_view(): + return "Protected content" + + with setup_app.test_request_context(): + # Mock unauthenticated user and LOGIN_DISABLED + mock_user = MockUser("test_user", is_authenticated=False) + with patch("libs.login._get_user", return_value=mock_user): + with patch("libs.login.dify_config") as mock_config: + mock_config.LOGIN_DISABLED = True + + result = protected_view() + assert result == "Protected content" + # Ensure unauthorized was not called + setup_app.login_manager.unauthorized.assert_not_called() + + def test_options_request_bypasses_authentication(self, setup_app: Flask): + """Test that OPTIONS requests are exempt from authentication.""" + + @login_required + def protected_view(): + return "Protected content" + + with setup_app.test_request_context(method="OPTIONS"): + # Mock unauthenticated user + mock_user = MockUser("test_user", is_authenticated=False) + with patch("libs.login._get_user", return_value=mock_user): + result = protected_view() + assert result == "Protected content" + # Ensure unauthorized was not called + setup_app.login_manager.unauthorized.assert_not_called() + + def test_flask_2_compatibility(self, setup_app: Flask): + """Test Flask 2.x compatibility with ensure_sync.""" + + @login_required + def protected_view(): + return "Protected content" + + # Mock Flask 2.x ensure_sync + setup_app.ensure_sync = MagicMock(return_value=lambda: "Synced content") + + with setup_app.test_request_context(): + mock_user = MockUser("test_user", is_authenticated=True) + with patch("libs.login._get_user", return_value=mock_user): + result = protected_view() + assert result == "Synced content" + setup_app.ensure_sync.assert_called_once() + + def test_flask_1_compatibility(self, setup_app: Flask): + """Test Flask 1.x compatibility without ensure_sync.""" + + @login_required + def protected_view(): + return "Protected content" + + # Remove ensure_sync to simulate Flask 1.x + if hasattr(setup_app, "ensure_sync"): + delattr(setup_app, "ensure_sync") + + with setup_app.test_request_context(): + mock_user = MockUser("test_user", is_authenticated=True) + with patch("libs.login._get_user", return_value=mock_user): + result = protected_view() + assert result == "Protected content" + + +class TestGetUser: + """Test cases for _get_user function.""" + + def test_get_user_returns_user_from_g(self, app: Flask): + """Test that _get_user returns user from g._login_user.""" + mock_user = MockUser("test_user") + + with app.test_request_context(): + g._login_user = mock_user + user = _get_user() + assert user == mock_user + assert user.id == "test_user" + + def test_get_user_loads_user_if_not_in_g(self, app: Flask): + """Test that _get_user loads user if not already in g.""" + mock_user = MockUser("test_user") + + # Mock login manager + login_manager = MagicMock() + login_manager._load_user = MagicMock() + app.login_manager = login_manager + + with app.test_request_context(): + # Simulate _load_user setting g._login_user + def side_effect(): + g._login_user = mock_user + + login_manager._load_user.side_effect = side_effect + + user = _get_user() + assert user == mock_user + login_manager._load_user.assert_called_once() + + def test_get_user_returns_none_without_request_context(self, app: Flask): + """Test that _get_user returns None outside request context.""" + # Outside of request context + user = _get_user() + assert user is None + + +class TestCurrentUser: + """Test cases for current_user proxy.""" + + def test_current_user_proxy_returns_authenticated_user(self, app: Flask): + """Test that current_user proxy returns authenticated user.""" + mock_user = MockUser("test_user", is_authenticated=True) + + with app.test_request_context(): + with patch("libs.login._get_user", return_value=mock_user): + assert current_user.id == "test_user" + assert current_user.is_authenticated is True + + def test_current_user_proxy_returns_none_when_no_user(self, app: Flask): + """Test that current_user proxy handles None user.""" + with app.test_request_context(): + with patch("libs.login._get_user", return_value=None): + # When _get_user returns None, accessing attributes should fail + # or current_user should evaluate to falsy + try: + # Try to access an attribute that would exist on a real user + _ = current_user.id + pytest.fail("Should have raised AttributeError") + except AttributeError: + # This is expected when current_user is None + pass + + def test_current_user_proxy_thread_safety(self, app: Flask): + """Test that current_user proxy is thread-safe.""" + import threading + + results = {} + + def check_user_in_thread(user_id: str, index: int): + with app.test_request_context(): + mock_user = MockUser(user_id) + with patch("libs.login._get_user", return_value=mock_user): + results[index] = current_user.id + + # Create multiple threads with different users + threads = [] + for i in range(5): + thread = threading.Thread(target=check_user_in_thread, args=(f"user_{i}", i)) + threads.append(thread) + thread.start() + + # Wait for all threads to complete + for thread in threads: + thread.join() + + # Verify each thread got its own user + for i in range(5): + assert results[i] == f"user_{i}" diff --git a/api/tests/unit_tests/libs/test_passport.py b/api/tests/unit_tests/libs/test_passport.py new file mode 100644 index 0000000000..f33484c18d --- /dev/null +++ b/api/tests/unit_tests/libs/test_passport.py @@ -0,0 +1,205 @@ +from datetime import UTC, datetime, timedelta +from unittest.mock import patch + +import jwt +import pytest +from werkzeug.exceptions import Unauthorized + +from libs.passport import PassportService + + +class TestPassportService: + """Test PassportService JWT operations""" + + @pytest.fixture + def passport_service(self): + """Create PassportService instance with test secret key""" + with patch("libs.passport.dify_config") as mock_config: + mock_config.SECRET_KEY = "test-secret-key-for-testing" + return PassportService() + + @pytest.fixture + def another_passport_service(self): + """Create another PassportService instance with different secret key""" + with patch("libs.passport.dify_config") as mock_config: + mock_config.SECRET_KEY = "another-secret-key-for-testing" + return PassportService() + + # Core functionality tests + def test_should_issue_and_verify_token(self, passport_service): + """Test complete JWT lifecycle: issue and verify""" + payload = {"user_id": "123", "app_code": "test-app"} + token = passport_service.issue(payload) + + # Verify token format + assert isinstance(token, str) + assert len(token.split(".")) == 3 # JWT format: header.payload.signature + + # Verify token content + decoded = passport_service.verify(token) + assert decoded == payload + + def test_should_handle_different_payload_types(self, passport_service): + """Test issuing and verifying tokens with different payload types""" + test_cases = [ + {"string": "value"}, + {"number": 42}, + {"float": 3.14}, + {"boolean": True}, + {"null": None}, + {"array": [1, 2, 3]}, + {"nested": {"key": "value"}}, + {"unicode": "中文测试"}, + {"emoji": "🔐"}, + {}, # Empty payload + ] + + for payload in test_cases: + token = passport_service.issue(payload) + decoded = passport_service.verify(token) + assert decoded == payload + + # Security tests + def test_should_reject_modified_token(self, passport_service): + """Test that any modification to token invalidates it""" + token = passport_service.issue({"user": "test"}) + + # Test multiple modification points + test_positions = [0, len(token) // 3, len(token) // 2, len(token) - 1] + + for pos in test_positions: + if pos < len(token) and token[pos] != ".": + # Change one character + tampered = token[:pos] + ("X" if token[pos] != "X" else "Y") + token[pos + 1 :] + with pytest.raises(Unauthorized): + passport_service.verify(tampered) + + def test_should_reject_token_with_different_secret_key(self, passport_service, another_passport_service): + """Test key isolation - token from one service should not work with another""" + payload = {"user_id": "123", "app_code": "test-app"} + token = passport_service.issue(payload) + + with pytest.raises(Unauthorized) as exc_info: + another_passport_service.verify(token) + assert str(exc_info.value) == "401 Unauthorized: Invalid token signature." + + def test_should_use_hs256_algorithm(self, passport_service): + """Test that HS256 algorithm is used for signing""" + payload = {"test": "data"} + token = passport_service.issue(payload) + + # Decode header without relying on JWT internals + # Use jwt.get_unverified_header which is a public API + header = jwt.get_unverified_header(token) + assert header["alg"] == "HS256" + + def test_should_reject_token_with_wrong_algorithm(self, passport_service): + """Test rejection of token signed with different algorithm""" + payload = {"user_id": "123"} + + # Create token with different algorithm + with patch("libs.passport.dify_config") as mock_config: + mock_config.SECRET_KEY = "test-secret-key-for-testing" + # Create token with HS512 instead of HS256 + wrong_alg_token = jwt.encode(payload, mock_config.SECRET_KEY, algorithm="HS512") + + # Should fail because service expects HS256 + # InvalidAlgorithmError is now caught by PyJWTError handler + with pytest.raises(Unauthorized) as exc_info: + passport_service.verify(wrong_alg_token) + assert str(exc_info.value) == "401 Unauthorized: Invalid token." + + # Exception handling tests + def test_should_handle_invalid_tokens(self, passport_service): + """Test handling of various invalid token formats""" + invalid_tokens = [ + ("not.a.token", "Invalid token."), + ("invalid-jwt-format", "Invalid token."), + ("xxx.yyy.zzz", "Invalid token."), + ("a.b", "Invalid token."), # Missing signature + ("", "Invalid token."), # Empty string + (" ", "Invalid token."), # Whitespace + (None, "Invalid token."), # None value + # Malformed base64 + ("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.INVALID_BASE64!@#$.signature", "Invalid token."), + ] + + for invalid_token, expected_message in invalid_tokens: + with pytest.raises(Unauthorized) as exc_info: + passport_service.verify(invalid_token) + assert expected_message in str(exc_info.value) + + def test_should_reject_expired_token(self, passport_service): + """Test rejection of expired token""" + past_time = datetime.now(UTC) - timedelta(hours=1) + payload = {"user_id": "123", "exp": past_time.timestamp()} + + with patch("libs.passport.dify_config") as mock_config: + mock_config.SECRET_KEY = "test-secret-key-for-testing" + token = jwt.encode(payload, mock_config.SECRET_KEY, algorithm="HS256") + + with pytest.raises(Unauthorized) as exc_info: + passport_service.verify(token) + assert str(exc_info.value) == "401 Unauthorized: Token has expired." + + # Configuration tests + def test_should_handle_empty_secret_key(self): + """Test behavior when SECRET_KEY is empty""" + with patch("libs.passport.dify_config") as mock_config: + mock_config.SECRET_KEY = "" + service = PassportService() + + # Empty secret key should still work but is insecure + payload = {"test": "data"} + token = service.issue(payload) + decoded = service.verify(token) + assert decoded == payload + + def test_should_handle_none_secret_key(self): + """Test behavior when SECRET_KEY is None""" + with patch("libs.passport.dify_config") as mock_config: + mock_config.SECRET_KEY = None + service = PassportService() + + payload = {"test": "data"} + # JWT library will raise TypeError when secret is None + with pytest.raises((TypeError, jwt.exceptions.InvalidKeyError)): + service.issue(payload) + + # Boundary condition tests + def test_should_handle_large_payload(self, passport_service): + """Test handling of large payload""" + # Test with 100KB instead of 1MB for faster tests + large_data = "x" * (100 * 1024) + payload = {"data": large_data} + + token = passport_service.issue(payload) + decoded = passport_service.verify(token) + + assert decoded["data"] == large_data + + def test_should_handle_special_characters_in_payload(self, passport_service): + """Test handling of special characters in payload""" + special_payloads = [ + {"special": "!@#$%^&*()"}, + {"quotes": 'He said "Hello"'}, + {"backslash": "path\\to\\file"}, + {"newline": "line1\nline2"}, + {"unicode": "🔐🔑🛡️"}, + {"mixed": "Test123!@#中文🔐"}, + ] + + for payload in special_payloads: + token = passport_service.issue(payload) + decoded = passport_service.verify(token) + assert decoded == payload + + def test_should_catch_generic_pyjwt_errors(self, passport_service): + """Test that generic PyJWTError exceptions are caught and converted to Unauthorized""" + # Mock jwt.decode to raise a generic PyJWTError + with patch("libs.passport.jwt.decode") as mock_decode: + mock_decode.side_effect = jwt.exceptions.PyJWTError("Generic JWT error") + + with pytest.raises(Unauthorized) as exc_info: + passport_service.verify("some-token") + assert str(exc_info.value) == "401 Unauthorized: Invalid token." diff --git a/api/tests/unit_tests/models/test_workflow.py b/api/tests/unit_tests/models/test_workflow.py index 69163d48bd..5bc77ad0ef 100644 --- a/api/tests/unit_tests/models/test_workflow.py +++ b/api/tests/unit_tests/models/test_workflow.py @@ -9,6 +9,7 @@ from core.file.models import File from core.variables import FloatVariable, IntegerVariable, SecretVariable, StringVariable from core.variables.segments import IntegerSegment, Segment from factories.variable_factory import build_segment +from models.model import EndUser from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel, is_system_variable_editable @@ -43,7 +44,7 @@ def test_environment_variables(): ) # Mock current_user as an EndUser - mock_user = mock.Mock() + mock_user = mock.Mock(spec=EndUser) mock_user.tenant_id = "tenant_id" with ( @@ -90,7 +91,7 @@ def test_update_environment_variables(): ) # Mock current_user as an EndUser - mock_user = mock.Mock() + mock_user = mock.Mock(spec=EndUser) mock_user.tenant_id = "tenant_id" with ( @@ -136,7 +137,7 @@ def test_to_dict(): # Create some EnvironmentVariable instances # Mock current_user as an EndUser - mock_user = mock.Mock() + mock_user = mock.Mock(spec=EndUser) mock_user.tenant_id = "tenant_id" with ( diff --git a/api/tests/unit_tests/services/services_test_help.py b/api/tests/unit_tests/services/services_test_help.py new file mode 100644 index 0000000000..c6b962f7fc --- /dev/null +++ b/api/tests/unit_tests/services/services_test_help.py @@ -0,0 +1,59 @@ +from unittest.mock import MagicMock + + +class ServiceDbTestHelper: + """ + Helper class for service database query tests. + """ + + @staticmethod + def setup_db_query_filter_by_mock(mock_db, query_results): + """ + Smart database query mock that responds based on model type and query parameters. + + Args: + mock_db: Mock database session + query_results: Dict mapping (model_name, filter_key, filter_value) to return value + Example: {('Account', 'email', 'test@example.com'): mock_account} + """ + + def query_side_effect(model): + mock_query = MagicMock() + + def filter_by_side_effect(**kwargs): + mock_filter_result = MagicMock() + + def first_side_effect(): + # Find matching result based on model and filter parameters + for (model_name, filter_key, filter_value), result in query_results.items(): + if model.__name__ == model_name and filter_key in kwargs and kwargs[filter_key] == filter_value: + return result + return None + + mock_filter_result.first.side_effect = first_side_effect + + # Handle order_by calls for complex queries + def order_by_side_effect(*args, **kwargs): + mock_order_result = MagicMock() + + def order_first_side_effect(): + # Look for order_by results in the same query_results dict + for (model_name, filter_key, filter_value), result in query_results.items(): + if ( + model.__name__ == model_name + and filter_key == "order_by" + and filter_value == "first_available" + ): + return result + return None + + mock_order_result.first.side_effect = order_first_side_effect + return mock_order_result + + mock_filter_result.order_by.side_effect = order_by_side_effect + return mock_filter_result + + mock_query.filter_by.side_effect = filter_by_side_effect + return mock_query + + mock_db.session.query.side_effect = query_side_effect diff --git a/api/tests/unit_tests/services/test_account_service.py b/api/tests/unit_tests/services/test_account_service.py new file mode 100644 index 0000000000..13900ab6d1 --- /dev/null +++ b/api/tests/unit_tests/services/test_account_service.py @@ -0,0 +1,1545 @@ +import json +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + +from configs import dify_config +from models.account import Account +from services.account_service import AccountService, RegisterService, TenantService +from services.errors.account import ( + AccountAlreadyInTenantError, + AccountLoginError, + AccountNotFoundError, + AccountPasswordError, + AccountRegisterError, + CurrentPasswordIncorrectError, +) +from tests.unit_tests.services.services_test_help import ServiceDbTestHelper + + +class TestAccountAssociatedDataFactory: + """Factory class for creating test data and mock objects for account service tests.""" + + @staticmethod + def create_account_mock( + account_id: str = "user-123", + email: str = "test@example.com", + name: str = "Test User", + status: str = "active", + password: str = "hashed_password", + password_salt: str = "salt", + interface_language: str = "en-US", + interface_theme: str = "light", + timezone: str = "UTC", + **kwargs, + ) -> MagicMock: + """Create a mock account with specified attributes.""" + account = MagicMock(spec=Account) + account.id = account_id + account.email = email + account.name = name + account.status = status + account.password = password + account.password_salt = password_salt + account.interface_language = interface_language + account.interface_theme = interface_theme + account.timezone = timezone + # Set last_active_at to a datetime object that's older than 10 minutes + account.last_active_at = datetime.now() - timedelta(minutes=15) + account.initialized_at = None + for key, value in kwargs.items(): + setattr(account, key, value) + return account + + @staticmethod + def create_tenant_join_mock( + tenant_id: str = "tenant-456", + account_id: str = "user-123", + current: bool = True, + role: str = "normal", + **kwargs, + ) -> MagicMock: + """Create a mock tenant account join record.""" + tenant_join = MagicMock() + tenant_join.tenant_id = tenant_id + tenant_join.account_id = account_id + tenant_join.current = current + tenant_join.role = role + for key, value in kwargs.items(): + setattr(tenant_join, key, value) + return tenant_join + + @staticmethod + def create_feature_service_mock(allow_register: bool = True): + """Create a mock feature service.""" + mock_service = MagicMock() + mock_service.get_system_features.return_value.is_allow_register = allow_register + return mock_service + + @staticmethod + def create_billing_service_mock(email_frozen: bool = False): + """Create a mock billing service.""" + mock_service = MagicMock() + mock_service.is_email_in_freeze.return_value = email_frozen + return mock_service + + +class TestAccountService: + """ + Comprehensive unit tests for AccountService methods. + + This test suite covers all account-related operations including: + - Authentication and login + - Account creation and registration + - Password management + - JWT token generation + - User loading and tenant management + - Error conditions and edge cases + """ + + @pytest.fixture + def mock_db_dependencies(self): + """Common mock setup for database dependencies.""" + with patch("services.account_service.db") as mock_db: + mock_db.session.add = MagicMock() + mock_db.session.commit = MagicMock() + yield { + "db": mock_db, + } + + @pytest.fixture + def mock_password_dependencies(self): + """Mock setup for password-related functions.""" + with ( + patch("services.account_service.compare_password") as mock_compare_password, + patch("services.account_service.hash_password") as mock_hash_password, + patch("services.account_service.valid_password") as mock_valid_password, + ): + yield { + "compare_password": mock_compare_password, + "hash_password": mock_hash_password, + "valid_password": mock_valid_password, + } + + @pytest.fixture + def mock_external_service_dependencies(self): + """Mock setup for external service dependencies.""" + with ( + patch("services.account_service.FeatureService") as mock_feature_service, + patch("services.account_service.BillingService") as mock_billing_service, + patch("services.account_service.PassportService") as mock_passport_service, + ): + yield { + "feature_service": mock_feature_service, + "billing_service": mock_billing_service, + "passport_service": mock_passport_service, + } + + @pytest.fixture + def mock_db_with_autospec(self): + """ + Mock database with autospec for more realistic behavior. + This approach preserves the actual method signatures and behavior. + """ + with patch("services.account_service.db", autospec=True) as mock_db: + # Create a more realistic session mock + mock_session = MagicMock() + mock_db.session = mock_session + + # Setup basic session methods + mock_session.add = MagicMock() + mock_session.commit = MagicMock() + mock_session.query = MagicMock() + + yield mock_db + + def _assert_database_operations_called(self, mock_db): + """Helper method to verify database operations were called.""" + mock_db.session.commit.assert_called() + + def _assert_database_operations_not_called(self, mock_db): + """Helper method to verify database operations were not called.""" + mock_db.session.commit.assert_not_called() + + def _assert_exception_raised(self, exception_type, callable_func, *args, **kwargs): + """Helper method to verify that specific exception is raised.""" + with pytest.raises(exception_type): + callable_func(*args, **kwargs) + + # ==================== Authentication Tests ==================== + + def test_authenticate_success(self, mock_db_dependencies, mock_password_dependencies): + """Test successful authentication with correct email and password.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + + # Setup smart database query mock + query_results = {("Account", "email", "test@example.com"): mock_account} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + mock_password_dependencies["compare_password"].return_value = True + + # Execute test + result = AccountService.authenticate("test@example.com", "password") + + # Verify results + assert result == mock_account + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_authenticate_account_not_found(self, mock_db_dependencies): + """Test authentication when account does not exist.""" + # Setup smart database query mock - no matching results + query_results = {("Account", "email", "notfound@example.com"): None} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Execute test and verify exception + self._assert_exception_raised( + AccountNotFoundError, AccountService.authenticate, "notfound@example.com", "password" + ) + + def test_authenticate_account_banned(self, mock_db_dependencies): + """Test authentication when account is banned.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock(status="banned") + + # Setup smart database query mock + query_results = {("Account", "email", "banned@example.com"): mock_account} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Execute test and verify exception + self._assert_exception_raised(AccountLoginError, AccountService.authenticate, "banned@example.com", "password") + + def test_authenticate_password_error(self, mock_db_dependencies, mock_password_dependencies): + """Test authentication with wrong password.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + + # Setup smart database query mock + query_results = {("Account", "email", "test@example.com"): mock_account} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + mock_password_dependencies["compare_password"].return_value = False + + # Execute test and verify exception + self._assert_exception_raised( + AccountPasswordError, AccountService.authenticate, "test@example.com", "wrongpassword" + ) + + def test_authenticate_pending_account_activates(self, mock_db_dependencies, mock_password_dependencies): + """Test authentication for a pending account, which should activate on login.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock(status="pending") + + # Setup smart database query mock + query_results = {("Account", "email", "pending@example.com"): mock_account} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + mock_password_dependencies["compare_password"].return_value = True + + # Execute test + result = AccountService.authenticate("pending@example.com", "password") + + # Verify results + assert result == mock_account + assert mock_account.status == "active" + self._assert_database_operations_called(mock_db_dependencies["db"]) + + # ==================== Account Creation Tests ==================== + + def test_create_account_success( + self, mock_db_dependencies, mock_password_dependencies, mock_external_service_dependencies + ): + """Test successful account creation with all required parameters.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + mock_password_dependencies["hash_password"].return_value = b"hashed_password" + + # Execute test + result = AccountService.create_account( + email="test@example.com", + name="Test User", + interface_language="en-US", + password="password123", + interface_theme="light", + ) + + # Verify results + assert result.email == "test@example.com" + assert result.name == "Test User" + assert result.interface_language == "en-US" + assert result.interface_theme == "light" + assert result.password is not None + assert result.password_salt is not None + assert result.timezone is not None + + # Verify database operations + mock_db_dependencies["db"].session.add.assert_called_once() + added_account = mock_db_dependencies["db"].session.add.call_args[0][0] + assert added_account.email == "test@example.com" + assert added_account.name == "Test User" + assert added_account.interface_language == "en-US" + assert added_account.interface_theme == "light" + assert added_account.password is not None + assert added_account.password_salt is not None + assert added_account.timezone is not None + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_create_account_registration_disabled(self, mock_external_service_dependencies): + """Test account creation when registration is disabled.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = False + + # Execute test and verify exception + self._assert_exception_raised( + Exception, # AccountNotFound + AccountService.create_account, + email="test@example.com", + name="Test User", + interface_language="en-US", + ) + + def test_create_account_email_frozen(self, mock_db_dependencies, mock_external_service_dependencies): + """Test account creation with frozen email address.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = True + dify_config.BILLING_ENABLED = True + + # Execute test and verify exception + self._assert_exception_raised( + AccountRegisterError, + AccountService.create_account, + email="frozen@example.com", + name="Test User", + interface_language="en-US", + ) + dify_config.BILLING_ENABLED = False + + def test_create_account_without_password(self, mock_db_dependencies, mock_external_service_dependencies): + """Test account creation without password (for invite-based registration).""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Execute test + result = AccountService.create_account( + email="test@example.com", + name="Test User", + interface_language="zh-CN", + password=None, + interface_theme="dark", + ) + + # Verify results + assert result.email == "test@example.com" + assert result.name == "Test User" + assert result.interface_language == "zh-CN" + assert result.interface_theme == "dark" + assert result.password is None + assert result.password_salt is None + assert result.timezone is not None + + # Verify database operations + mock_db_dependencies["db"].session.add.assert_called_once() + added_account = mock_db_dependencies["db"].session.add.call_args[0][0] + assert added_account.email == "test@example.com" + assert added_account.name == "Test User" + assert added_account.interface_language == "zh-CN" + assert added_account.interface_theme == "dark" + assert added_account.password is None + assert added_account.password_salt is None + assert added_account.timezone is not None + self._assert_database_operations_called(mock_db_dependencies["db"]) + + # ==================== Password Management Tests ==================== + + def test_update_account_password_success(self, mock_db_dependencies, mock_password_dependencies): + """Test successful password update with correct current password and valid new password.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + mock_password_dependencies["compare_password"].return_value = True + mock_password_dependencies["valid_password"].return_value = None + mock_password_dependencies["hash_password"].return_value = b"new_hashed_password" + + # Execute test + result = AccountService.update_account_password(mock_account, "old_password", "new_password123") + + # Verify results + assert result == mock_account + assert mock_account.password is not None + assert mock_account.password_salt is not None + + # Verify password validation was called + mock_password_dependencies["compare_password"].assert_called_once_with( + "old_password", "hashed_password", "salt" + ) + mock_password_dependencies["valid_password"].assert_called_once_with("new_password123") + + # Verify database operations + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_update_account_password_current_password_incorrect(self, mock_password_dependencies): + """Test password update with incorrect current password.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + mock_password_dependencies["compare_password"].return_value = False + + # Execute test and verify exception + self._assert_exception_raised( + CurrentPasswordIncorrectError, + AccountService.update_account_password, + mock_account, + "wrong_password", + "new_password123", + ) + + # Verify password comparison was called + mock_password_dependencies["compare_password"].assert_called_once_with( + "wrong_password", "hashed_password", "salt" + ) + + def test_update_account_password_invalid_new_password(self, mock_password_dependencies): + """Test password update with invalid new password.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + mock_password_dependencies["compare_password"].return_value = True + mock_password_dependencies["valid_password"].side_effect = ValueError("Password too short") + + # Execute test and verify exception + self._assert_exception_raised( + ValueError, AccountService.update_account_password, mock_account, "old_password", "short" + ) + + # Verify password validation was called + mock_password_dependencies["valid_password"].assert_called_once_with("short") + + # ==================== User Loading Tests ==================== + + def test_load_user_success(self, mock_db_dependencies): + """Test successful user loading with current tenant.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + mock_tenant_join = TestAccountAssociatedDataFactory.create_tenant_join_mock() + + # Setup smart database query mock + query_results = { + ("Account", "id", "user-123"): mock_account, + ("TenantAccountJoin", "account_id", "user-123"): mock_tenant_join, + } + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Mock datetime + with patch("services.account_service.datetime") as mock_datetime: + mock_now = datetime.now() + mock_datetime.now.return_value = mock_now + mock_datetime.UTC = "UTC" + + # Execute test + result = AccountService.load_user("user-123") + + # Verify results + assert result == mock_account + assert mock_account.set_tenant_id.called + + def test_load_user_not_found(self, mock_db_dependencies): + """Test user loading when user does not exist.""" + # Setup smart database query mock - no matching results + query_results = {("Account", "id", "non-existent-user"): None} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Execute test + result = AccountService.load_user("non-existent-user") + + # Verify results + assert result is None + + def test_load_user_banned(self, mock_db_dependencies): + """Test user loading when user is banned.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock(status="banned") + + # Setup smart database query mock + query_results = {("Account", "id", "user-123"): mock_account} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Execute test and verify exception + self._assert_exception_raised( + Exception, # Unauthorized + AccountService.load_user, + "user-123", + ) + + def test_load_user_no_current_tenant(self, mock_db_dependencies): + """Test user loading when user has no current tenant but has available tenants.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + mock_available_tenant = TestAccountAssociatedDataFactory.create_tenant_join_mock(current=False) + + # Setup smart database query mock for complex scenario + query_results = { + ("Account", "id", "user-123"): mock_account, + ("TenantAccountJoin", "account_id", "user-123"): None, # No current tenant + ("TenantAccountJoin", "order_by", "first_available"): mock_available_tenant, # First available tenant + } + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Mock datetime + with patch("services.account_service.datetime") as mock_datetime: + mock_now = datetime.now() + mock_datetime.now.return_value = mock_now + mock_datetime.UTC = "UTC" + + # Execute test + result = AccountService.load_user("user-123") + + # Verify results + assert result == mock_account + assert mock_available_tenant.current is True + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_load_user_no_tenants(self, mock_db_dependencies): + """Test user loading when user has no tenants at all.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + + # Setup smart database query mock for no tenants scenario + query_results = { + ("Account", "id", "user-123"): mock_account, + ("TenantAccountJoin", "account_id", "user-123"): None, # No current tenant + ("TenantAccountJoin", "order_by", "first_available"): None, # No available tenants + } + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Mock datetime + with patch("services.account_service.datetime") as mock_datetime: + mock_now = datetime.now() + mock_datetime.now.return_value = mock_now + mock_datetime.UTC = "UTC" + + # Execute test + result = AccountService.load_user("user-123") + + # Verify results + assert result is None + + +class TestTenantService: + """ + Comprehensive unit tests for TenantService methods. + + This test suite covers all tenant-related operations including: + - Tenant creation and management + - Member management and permissions + - Tenant switching + - Role updates and permission checks + - Error conditions and edge cases + """ + + @pytest.fixture + def mock_db_dependencies(self): + """Common mock setup for database dependencies.""" + with patch("services.account_service.db") as mock_db: + mock_db.session.add = MagicMock() + mock_db.session.commit = MagicMock() + yield { + "db": mock_db, + } + + @pytest.fixture + def mock_rsa_dependencies(self): + """Mock setup for RSA-related functions.""" + with patch("services.account_service.generate_key_pair") as mock_generate_key_pair: + yield mock_generate_key_pair + + @pytest.fixture + def mock_external_service_dependencies(self): + """Mock setup for external service dependencies.""" + with ( + patch("services.account_service.FeatureService") as mock_feature_service, + patch("services.account_service.BillingService") as mock_billing_service, + ): + yield { + "feature_service": mock_feature_service, + "billing_service": mock_billing_service, + } + + def _assert_database_operations_called(self, mock_db): + """Helper method to verify database operations were called.""" + mock_db.session.commit.assert_called() + + def _assert_exception_raised(self, exception_type, callable_func, *args, **kwargs): + """Helper method to verify that specific exception is raised.""" + with pytest.raises(exception_type): + callable_func(*args, **kwargs) + + # ==================== Tenant Creation Tests ==================== + + def test_create_owner_tenant_if_not_exist_new_user( + self, mock_db_dependencies, mock_rsa_dependencies, mock_external_service_dependencies + ): + """Test creating owner tenant for new user without existing tenants.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + + # Setup smart database query mock - no existing tenant joins + query_results = { + ("TenantAccountJoin", "account_id", "user-123"): None, + ("TenantAccountJoin", "tenant_id", "tenant-456"): None, # For has_roles check + } + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Setup external service mocks + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.is_allow_create_workspace = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.license.workspaces.is_available.return_value = True + + # Mock tenant creation + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_tenant.name = "Test User's Workspace" + + # Mock database operations + mock_db_dependencies["db"].session.add = MagicMock() + + # Mock RSA key generation + mock_rsa_dependencies.return_value = "mock_public_key" + + # Mock has_roles method to return False (no existing owner) + with patch("services.account_service.TenantService.has_roles") as mock_has_roles: + mock_has_roles.return_value = False + + # Mock Tenant creation to set proper ID + with patch("services.account_service.Tenant") as mock_tenant_class: + mock_tenant_instance = MagicMock() + mock_tenant_instance.id = "tenant-456" + mock_tenant_instance.name = "Test User's Workspace" + mock_tenant_class.return_value = mock_tenant_instance + + # Execute test + TenantService.create_owner_tenant_if_not_exist(mock_account) + + # Verify tenant was created with correct parameters + mock_db_dependencies["db"].session.add.assert_called() + + # Get all calls to session.add + add_calls = mock_db_dependencies["db"].session.add.call_args_list + + # Should have at least 2 calls: one for Tenant, one for TenantAccountJoin + assert len(add_calls) >= 2 + + # Verify Tenant was added with correct name + tenant_added = False + tenant_account_join_added = False + + for call in add_calls: + added_object = call[0][0] # First argument of the call + + # Check if it's a Tenant object + if hasattr(added_object, "name") and hasattr(added_object, "id"): + # This should be a Tenant object + assert added_object.name == "Test User's Workspace" + tenant_added = True + + # Check if it's a TenantAccountJoin object + elif ( + hasattr(added_object, "tenant_id") + and hasattr(added_object, "account_id") + and hasattr(added_object, "role") + ): + # This should be a TenantAccountJoin object + assert added_object.tenant_id is not None + assert added_object.account_id == "user-123" + assert added_object.role == "owner" + tenant_account_join_added = True + + assert tenant_added, "Tenant object was not added to database" + assert tenant_account_join_added, "TenantAccountJoin object was not added to database" + + self._assert_database_operations_called(mock_db_dependencies["db"]) + assert mock_rsa_dependencies.called, "RSA key generation was not called" + + # ==================== Member Management Tests ==================== + + def test_create_tenant_member_success(self, mock_db_dependencies): + """Test successful tenant member creation.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + + # Setup smart database query mock - no existing member + query_results = {("TenantAccountJoin", "tenant_id", "tenant-456"): None} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Mock database operations + mock_db_dependencies["db"].session.add = MagicMock() + + # Execute test + result = TenantService.create_tenant_member(mock_tenant, mock_account, "normal") + + # Verify member was created with correct parameters + assert result is not None + mock_db_dependencies["db"].session.add.assert_called_once() + + # Verify the TenantAccountJoin object was added with correct parameters + added_tenant_account_join = mock_db_dependencies["db"].session.add.call_args[0][0] + assert added_tenant_account_join.tenant_id == "tenant-456" + assert added_tenant_account_join.account_id == "user-123" + assert added_tenant_account_join.role == "normal" + + self._assert_database_operations_called(mock_db_dependencies["db"]) + + # ==================== Tenant Switching Tests ==================== + + def test_switch_tenant_success(self): + """Test successful tenant switching.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + mock_tenant_join = TestAccountAssociatedDataFactory.create_tenant_join_mock( + tenant_id="tenant-456", account_id="user-123", current=False + ) + + # Mock the complex query in switch_tenant method + with patch("services.account_service.db") as mock_db: + # Mock the join query that returns the tenant_account_join + mock_query = MagicMock() + mock_filter = MagicMock() + mock_filter.first.return_value = mock_tenant_join + mock_query.filter.return_value = mock_filter + mock_query.join.return_value = mock_query + mock_db.session.query.return_value = mock_query + + # Execute test + TenantService.switch_tenant(mock_account, "tenant-456") + + # Verify tenant was switched + assert mock_tenant_join.current is True + self._assert_database_operations_called(mock_db) + + def test_switch_tenant_no_tenant_id(self): + """Test tenant switching without providing tenant ID.""" + # Setup test data + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + + # Execute test and verify exception + self._assert_exception_raised(ValueError, TenantService.switch_tenant, mock_account, None) + + # ==================== Role Management Tests ==================== + + def test_update_member_role_success(self): + """Test successful member role update.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_member = TestAccountAssociatedDataFactory.create_account_mock(account_id="member-789") + mock_operator = TestAccountAssociatedDataFactory.create_account_mock(account_id="operator-123") + mock_target_join = TestAccountAssociatedDataFactory.create_tenant_join_mock( + tenant_id="tenant-456", account_id="member-789", role="normal" + ) + mock_operator_join = TestAccountAssociatedDataFactory.create_tenant_join_mock( + tenant_id="tenant-456", account_id="operator-123", role="owner" + ) + + # Mock the database queries in update_member_role method + with patch("services.account_service.db") as mock_db: + # Mock the first query for operator permission check + mock_query1 = MagicMock() + mock_filter1 = MagicMock() + mock_filter1.first.return_value = mock_operator_join + mock_query1.filter_by.return_value = mock_filter1 + + # Mock the second query for target member + mock_query2 = MagicMock() + mock_filter2 = MagicMock() + mock_filter2.first.return_value = mock_target_join + mock_query2.filter_by.return_value = mock_filter2 + + # Make the query method return different mocks for different calls + mock_db.session.query.side_effect = [mock_query1, mock_query2] + + # Execute test + TenantService.update_member_role(mock_tenant, mock_member, "admin", mock_operator) + + # Verify role was updated + assert mock_target_join.role == "admin" + self._assert_database_operations_called(mock_db) + + # ==================== Permission Check Tests ==================== + + def test_check_member_permission_success(self, mock_db_dependencies): + """Test successful member permission check.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_operator = TestAccountAssociatedDataFactory.create_account_mock(account_id="operator-123") + mock_member = TestAccountAssociatedDataFactory.create_account_mock(account_id="member-789") + mock_operator_join = TestAccountAssociatedDataFactory.create_tenant_join_mock( + tenant_id="tenant-456", account_id="operator-123", role="owner" + ) + + # Setup smart database query mock + query_results = {("TenantAccountJoin", "tenant_id", "tenant-456"): mock_operator_join} + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Execute test - should not raise exception + TenantService.check_member_permission(mock_tenant, mock_operator, mock_member, "add") + + def test_check_member_permission_operate_self(self): + """Test member permission check when operator tries to operate self.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_operator = TestAccountAssociatedDataFactory.create_account_mock(account_id="operator-123") + + # Execute test and verify exception + from services.errors.account import CannotOperateSelfError + + self._assert_exception_raised( + CannotOperateSelfError, + TenantService.check_member_permission, + mock_tenant, + mock_operator, + mock_operator, # Same as operator + "add", + ) + + +class TestRegisterService: + """ + Comprehensive unit tests for RegisterService methods. + + This test suite covers all registration-related operations including: + - System setup + - Account registration + - Member invitation + - Token management + - Invitation validation + - Error conditions and edge cases + """ + + @pytest.fixture + def mock_db_dependencies(self): + """Common mock setup for database dependencies.""" + with patch("services.account_service.db") as mock_db: + mock_db.session.add = MagicMock() + mock_db.session.commit = MagicMock() + mock_db.session.begin_nested = MagicMock() + mock_db.session.rollback = MagicMock() + yield { + "db": mock_db, + } + + @pytest.fixture + def mock_redis_dependencies(self): + """Mock setup for Redis-related functions.""" + with patch("services.account_service.redis_client") as mock_redis: + yield mock_redis + + @pytest.fixture + def mock_external_service_dependencies(self): + """Mock setup for external service dependencies.""" + with ( + patch("services.account_service.FeatureService") as mock_feature_service, + patch("services.account_service.BillingService") as mock_billing_service, + patch("services.account_service.PassportService") as mock_passport_service, + ): + yield { + "feature_service": mock_feature_service, + "billing_service": mock_billing_service, + "passport_service": mock_passport_service, + } + + @pytest.fixture + def mock_task_dependencies(self): + """Mock setup for task dependencies.""" + with patch("services.account_service.send_invite_member_mail_task") as mock_send_mail: + yield mock_send_mail + + def _assert_database_operations_called(self, mock_db): + """Helper method to verify database operations were called.""" + mock_db.session.commit.assert_called() + + def _assert_database_operations_not_called(self, mock_db): + """Helper method to verify database operations were not called.""" + mock_db.session.commit.assert_not_called() + + def _assert_exception_raised(self, exception_type, callable_func, *args, **kwargs): + """Helper method to verify that specific exception is raised.""" + with pytest.raises(exception_type): + callable_func(*args, **kwargs) + + # ==================== Setup Tests ==================== + + def test_setup_success(self, mock_db_dependencies, mock_external_service_dependencies): + """Test successful system setup.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + with patch("services.account_service.AccountService.create_account") as mock_create_account: + mock_create_account.return_value = mock_account + + # Mock TenantService.create_owner_tenant_if_not_exist + with patch("services.account_service.TenantService.create_owner_tenant_if_not_exist") as mock_create_tenant: + # Mock DifySetup + with patch("services.account_service.DifySetup") as mock_dify_setup: + mock_dify_setup_instance = MagicMock() + mock_dify_setup.return_value = mock_dify_setup_instance + + # Execute test + RegisterService.setup("admin@example.com", "Admin User", "password123", "192.168.1.1") + + # Verify results + mock_create_account.assert_called_once_with( + email="admin@example.com", + name="Admin User", + interface_language="en-US", + password="password123", + is_setup=True, + ) + mock_create_tenant.assert_called_once_with(account=mock_account, is_setup=True) + mock_dify_setup.assert_called_once() + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_setup_failure_rollback(self, mock_db_dependencies, mock_external_service_dependencies): + """Test setup failure with proper rollback.""" + # Setup mocks to simulate failure + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account to raise exception + with patch("services.account_service.AccountService.create_account") as mock_create_account: + mock_create_account.side_effect = Exception("Database error") + + # Execute test and verify exception + self._assert_exception_raised( + ValueError, + RegisterService.setup, + "admin@example.com", + "Admin User", + "password123", + "192.168.1.1", + ) + + # Verify rollback operations were called + mock_db_dependencies["db"].session.query.assert_called() + + # ==================== Registration Tests ==================== + + def test_register_success(self, mock_db_dependencies, mock_external_service_dependencies): + """Test successful account registration.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.is_allow_create_workspace = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.license.workspaces.is_available.return_value = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + with patch("services.account_service.AccountService.create_account") as mock_create_account: + mock_create_account.return_value = mock_account + + # Mock TenantService.create_tenant and create_tenant_member + with ( + patch("services.account_service.TenantService.create_tenant") as mock_create_tenant, + patch("services.account_service.TenantService.create_tenant_member") as mock_create_member, + patch("services.account_service.tenant_was_created") as mock_event, + ): + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_create_tenant.return_value = mock_tenant + + # Execute test + result = RegisterService.register( + email="test@example.com", + name="Test User", + password="password123", + language="en-US", + ) + + # Verify results + assert result == mock_account + assert result.status == "active" + assert result.initialized_at is not None + mock_create_account.assert_called_once_with( + email="test@example.com", + name="Test User", + interface_language="en-US", + password="password123", + is_setup=False, + ) + mock_create_tenant.assert_called_once_with("Test User's Workspace") + mock_create_member.assert_called_once_with(mock_tenant, mock_account, role="owner") + mock_event.send.assert_called_once_with(mock_tenant) + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_register_with_oauth(self, mock_db_dependencies, mock_external_service_dependencies): + """Test account registration with OAuth integration.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.is_allow_create_workspace = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.license.workspaces.is_available.return_value = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account and link_account_integrate + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + with ( + patch("services.account_service.AccountService.create_account") as mock_create_account, + patch("services.account_service.AccountService.link_account_integrate") as mock_link_account, + ): + mock_create_account.return_value = mock_account + + # Mock TenantService methods + with ( + patch("services.account_service.TenantService.create_tenant") as mock_create_tenant, + patch("services.account_service.TenantService.create_tenant_member") as mock_create_member, + patch("services.account_service.tenant_was_created") as mock_event, + ): + mock_tenant = MagicMock() + mock_create_tenant.return_value = mock_tenant + + # Execute test + result = RegisterService.register( + email="test@example.com", + name="Test User", + password=None, + open_id="oauth123", + provider="google", + language="en-US", + ) + + # Verify results + assert result == mock_account + mock_link_account.assert_called_once_with("google", "oauth123", mock_account) + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_register_with_pending_status(self, mock_db_dependencies, mock_external_service_dependencies): + """Test account registration with pending status.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.is_allow_create_workspace = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.license.workspaces.is_available.return_value = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + with patch("services.account_service.AccountService.create_account") as mock_create_account: + mock_create_account.return_value = mock_account + + # Mock TenantService methods + with ( + patch("services.account_service.TenantService.create_tenant") as mock_create_tenant, + patch("services.account_service.TenantService.create_tenant_member") as mock_create_member, + patch("services.account_service.tenant_was_created") as mock_event, + ): + mock_tenant = MagicMock() + mock_create_tenant.return_value = mock_tenant + + # Execute test with pending status + from models.account import AccountStatus + + result = RegisterService.register( + email="test@example.com", + name="Test User", + password="password123", + language="en-US", + status=AccountStatus.PENDING, + ) + + # Verify results + assert result == mock_account + assert result.status == "pending" + self._assert_database_operations_called(mock_db_dependencies["db"]) + + def test_register_workspace_not_allowed(self, mock_db_dependencies, mock_external_service_dependencies): + """Test registration when workspace creation is not allowed.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.is_allow_create_workspace = True + mock_external_service_dependencies[ + "feature_service" + ].get_system_features.return_value.license.workspaces.is_available.return_value = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account + mock_account = TestAccountAssociatedDataFactory.create_account_mock() + with patch("services.account_service.AccountService.create_account") as mock_create_account: + mock_create_account.return_value = mock_account + + # Execute test and verify exception + from services.errors.workspace import WorkSpaceNotAllowedCreateError + + with patch("services.account_service.TenantService.create_tenant") as mock_create_tenant: + mock_create_tenant.side_effect = WorkSpaceNotAllowedCreateError() + + self._assert_exception_raised( + AccountRegisterError, + RegisterService.register, + email="test@example.com", + name="Test User", + password="password123", + language="en-US", + ) + + # Verify rollback was called + mock_db_dependencies["db"].session.rollback.assert_called() + + def test_register_general_exception(self, mock_db_dependencies, mock_external_service_dependencies): + """Test registration with general exception handling.""" + # Setup mocks + mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True + mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False + + # Mock AccountService.create_account to raise exception + with patch("services.account_service.AccountService.create_account") as mock_create_account: + mock_create_account.side_effect = Exception("Unexpected error") + + # Execute test and verify exception + self._assert_exception_raised( + AccountRegisterError, + RegisterService.register, + email="test@example.com", + name="Test User", + password="password123", + language="en-US", + ) + + # Verify rollback was called + mock_db_dependencies["db"].session.rollback.assert_called() + + # ==================== Member Invitation Tests ==================== + + def test_invite_new_member_new_account(self, mock_db_dependencies, mock_redis_dependencies, mock_task_dependencies): + """Test inviting a new member who doesn't have an account.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_tenant.name = "Test Workspace" + mock_inviter = TestAccountAssociatedDataFactory.create_account_mock(account_id="inviter-123", name="Inviter") + + # Mock database queries - need to mock the Session query + mock_session = MagicMock() + mock_session.query.return_value.filter_by.return_value.first.return_value = None # No existing account + + with patch("services.account_service.Session") as mock_session_class: + mock_session_class.return_value.__enter__.return_value = mock_session + mock_session_class.return_value.__exit__.return_value = None + + # Mock RegisterService.register + mock_new_account = TestAccountAssociatedDataFactory.create_account_mock( + account_id="new-user-456", email="newuser@example.com", name="newuser", status="pending" + ) + with patch("services.account_service.RegisterService.register") as mock_register: + mock_register.return_value = mock_new_account + + # Mock TenantService methods + with ( + patch("services.account_service.TenantService.check_member_permission") as mock_check_permission, + patch("services.account_service.TenantService.create_tenant_member") as mock_create_member, + patch("services.account_service.TenantService.switch_tenant") as mock_switch_tenant, + patch("services.account_service.RegisterService.generate_invite_token") as mock_generate_token, + ): + mock_generate_token.return_value = "invite-token-123" + + # Execute test + result = RegisterService.invite_new_member( + tenant=mock_tenant, + email="newuser@example.com", + language="en-US", + role="normal", + inviter=mock_inviter, + ) + + # Verify results + assert result == "invite-token-123" + mock_register.assert_called_once_with( + email="newuser@example.com", + name="newuser", + language="en-US", + status="pending", + is_setup=True, + ) + mock_create_member.assert_called_once_with(mock_tenant, mock_new_account, "normal") + mock_switch_tenant.assert_called_once_with(mock_new_account, mock_tenant.id) + mock_generate_token.assert_called_once_with(mock_tenant, mock_new_account) + mock_task_dependencies.delay.assert_called_once() + + def test_invite_new_member_existing_account( + self, mock_db_dependencies, mock_redis_dependencies, mock_task_dependencies + ): + """Test inviting a new member who already has an account.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_tenant.name = "Test Workspace" + mock_inviter = TestAccountAssociatedDataFactory.create_account_mock(account_id="inviter-123", name="Inviter") + mock_existing_account = TestAccountAssociatedDataFactory.create_account_mock( + account_id="existing-user-456", email="existing@example.com", status="pending" + ) + + # Mock database queries - need to mock the Session query + mock_session = MagicMock() + mock_session.query.return_value.filter_by.return_value.first.return_value = mock_existing_account + + with patch("services.account_service.Session") as mock_session_class: + mock_session_class.return_value.__enter__.return_value = mock_session + mock_session_class.return_value.__exit__.return_value = None + + # Mock the db.session.query for TenantAccountJoin + mock_db_query = MagicMock() + mock_db_query.filter_by.return_value.first.return_value = None # No existing member + mock_db_dependencies["db"].session.query.return_value = mock_db_query + + # Mock TenantService methods + with ( + patch("services.account_service.TenantService.check_member_permission") as mock_check_permission, + patch("services.account_service.TenantService.create_tenant_member") as mock_create_member, + patch("services.account_service.RegisterService.generate_invite_token") as mock_generate_token, + ): + mock_generate_token.return_value = "invite-token-123" + + # Execute test + result = RegisterService.invite_new_member( + tenant=mock_tenant, + email="existing@example.com", + language="en-US", + role="normal", + inviter=mock_inviter, + ) + + # Verify results + assert result == "invite-token-123" + mock_create_member.assert_called_once_with(mock_tenant, mock_existing_account, "normal") + mock_generate_token.assert_called_once_with(mock_tenant, mock_existing_account) + mock_task_dependencies.delay.assert_called_once() + + def test_invite_new_member_already_in_tenant(self, mock_db_dependencies, mock_redis_dependencies): + """Test inviting a member who is already in the tenant.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_inviter = TestAccountAssociatedDataFactory.create_account_mock(account_id="inviter-123", name="Inviter") + mock_existing_account = TestAccountAssociatedDataFactory.create_account_mock( + account_id="existing-user-456", email="existing@example.com", status="active" + ) + + # Mock database queries + query_results = { + ("Account", "email", "existing@example.com"): mock_existing_account, + ( + "TenantAccountJoin", + "tenant_id", + "tenant-456", + ): TestAccountAssociatedDataFactory.create_tenant_join_mock(), + } + ServiceDbTestHelper.setup_db_query_filter_by_mock(mock_db_dependencies["db"], query_results) + + # Mock TenantService methods + with patch("services.account_service.TenantService.check_member_permission") as mock_check_permission: + # Execute test and verify exception + self._assert_exception_raised( + AccountAlreadyInTenantError, + RegisterService.invite_new_member, + tenant=mock_tenant, + email="existing@example.com", + language="en-US", + role="normal", + inviter=mock_inviter, + ) + + def test_invite_new_member_no_inviter(self): + """Test inviting a member without providing an inviter.""" + # Setup test data + mock_tenant = MagicMock() + + # Execute test and verify exception + self._assert_exception_raised( + ValueError, + RegisterService.invite_new_member, + tenant=mock_tenant, + email="test@example.com", + language="en-US", + role="normal", + inviter=None, + ) + + # ==================== Token Management Tests ==================== + + def test_generate_invite_token_success(self, mock_redis_dependencies): + """Test successful invite token generation.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_account = TestAccountAssociatedDataFactory.create_account_mock( + account_id="user-123", email="test@example.com" + ) + + # Mock uuid generation + with patch("services.account_service.uuid.uuid4") as mock_uuid: + mock_uuid.return_value = "test-uuid-123" + + # Execute test + result = RegisterService.generate_invite_token(mock_tenant, mock_account) + + # Verify results + assert result == "test-uuid-123" + mock_redis_dependencies.setex.assert_called_once() + + # Verify the stored data + call_args = mock_redis_dependencies.setex.call_args + assert call_args[0][0] == "member_invite:token:test-uuid-123" + stored_data = json.loads(call_args[0][2]) + assert stored_data["account_id"] == "user-123" + assert stored_data["email"] == "test@example.com" + assert stored_data["workspace_id"] == "tenant-456" + + def test_is_valid_invite_token_valid(self, mock_redis_dependencies): + """Test checking valid invite token.""" + # Setup mock + mock_redis_dependencies.get.return_value = b'{"test": "data"}' + + # Execute test + result = RegisterService.is_valid_invite_token("valid-token") + + # Verify results + assert result is True + mock_redis_dependencies.get.assert_called_once_with("member_invite:token:valid-token") + + def test_is_valid_invite_token_invalid(self, mock_redis_dependencies): + """Test checking invalid invite token.""" + # Setup mock + mock_redis_dependencies.get.return_value = None + + # Execute test + result = RegisterService.is_valid_invite_token("invalid-token") + + # Verify results + assert result is False + mock_redis_dependencies.get.assert_called_once_with("member_invite:token:invalid-token") + + def test_revoke_token_with_workspace_and_email(self, mock_redis_dependencies): + """Test revoking token with workspace ID and email.""" + # Execute test + RegisterService.revoke_token("workspace-123", "test@example.com", "token-123") + + # Verify results + mock_redis_dependencies.delete.assert_called_once() + call_args = mock_redis_dependencies.delete.call_args + assert "workspace-123" in call_args[0][0] + # The email is hashed, so we check for the hash pattern instead + assert "member_invite_token:" in call_args[0][0] + + def test_revoke_token_without_workspace_and_email(self, mock_redis_dependencies): + """Test revoking token without workspace ID and email.""" + # Execute test + RegisterService.revoke_token("", "", "token-123") + + # Verify results + mock_redis_dependencies.delete.assert_called_once_with("member_invite:token:token-123") + + # ==================== Invitation Validation Tests ==================== + + def test_get_invitation_if_token_valid_success(self, mock_db_dependencies, mock_redis_dependencies): + """Test successful invitation validation.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_tenant.status = "normal" + mock_account = TestAccountAssociatedDataFactory.create_account_mock( + account_id="user-123", email="test@example.com" + ) + + with patch("services.account_service.RegisterService._get_invitation_by_token") as mock_get_invitation_by_token: + # Mock the invitation data returned by _get_invitation_by_token + invitation_data = { + "account_id": "user-123", + "email": "test@example.com", + "workspace_id": "tenant-456", + } + mock_get_invitation_by_token.return_value = invitation_data + + # Mock database queries - complex query mocking + mock_query1 = MagicMock() + mock_query1.filter.return_value.first.return_value = mock_tenant + + mock_query2 = MagicMock() + mock_query2.join.return_value.filter.return_value.first.return_value = (mock_account, "normal") + + mock_db_dependencies["db"].session.query.side_effect = [mock_query1, mock_query2] + + # Execute test + result = RegisterService.get_invitation_if_token_valid("tenant-456", "test@example.com", "token-123") + + # Verify results + assert result is not None + assert result["account"] == mock_account + assert result["tenant"] == mock_tenant + assert result["data"] == invitation_data + + def test_get_invitation_if_token_valid_no_token_data(self, mock_redis_dependencies): + """Test invitation validation with no token data.""" + # Setup mock + mock_redis_dependencies.get.return_value = None + + # Execute test + result = RegisterService.get_invitation_if_token_valid("tenant-456", "test@example.com", "token-123") + + # Verify results + assert result is None + + def test_get_invitation_if_token_valid_tenant_not_found(self, mock_db_dependencies, mock_redis_dependencies): + """Test invitation validation when tenant is not found.""" + # Setup mock Redis data + invitation_data = { + "account_id": "user-123", + "email": "test@example.com", + "workspace_id": "tenant-456", + } + mock_redis_dependencies.get.return_value = json.dumps(invitation_data).encode() + + # Mock database queries - no tenant found + mock_query = MagicMock() + mock_query.filter.return_value.first.return_value = None + mock_db_dependencies["db"].session.query.return_value = mock_query + + # Execute test + result = RegisterService.get_invitation_if_token_valid("tenant-456", "test@example.com", "token-123") + + # Verify results + assert result is None + + def test_get_invitation_if_token_valid_account_not_found(self, mock_db_dependencies, mock_redis_dependencies): + """Test invitation validation when account is not found.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_tenant.status = "normal" + + # Mock Redis data + invitation_data = { + "account_id": "user-123", + "email": "test@example.com", + "workspace_id": "tenant-456", + } + mock_redis_dependencies.get.return_value = json.dumps(invitation_data).encode() + + # Mock database queries + mock_query1 = MagicMock() + mock_query1.filter.return_value.first.return_value = mock_tenant + + mock_query2 = MagicMock() + mock_query2.join.return_value.filter.return_value.first.return_value = None # No account found + + mock_db_dependencies["db"].session.query.side_effect = [mock_query1, mock_query2] + + # Execute test + result = RegisterService.get_invitation_if_token_valid("tenant-456", "test@example.com", "token-123") + + # Verify results + assert result is None + + def test_get_invitation_if_token_valid_account_id_mismatch(self, mock_db_dependencies, mock_redis_dependencies): + """Test invitation validation when account ID doesn't match.""" + # Setup test data + mock_tenant = MagicMock() + mock_tenant.id = "tenant-456" + mock_tenant.status = "normal" + mock_account = TestAccountAssociatedDataFactory.create_account_mock( + account_id="different-user-456", email="test@example.com" + ) + + # Mock Redis data with different account ID + invitation_data = { + "account_id": "user-123", + "email": "test@example.com", + "workspace_id": "tenant-456", + } + mock_redis_dependencies.get.return_value = json.dumps(invitation_data).encode() + + # Mock database queries + mock_query1 = MagicMock() + mock_query1.filter.return_value.first.return_value = mock_tenant + + mock_query2 = MagicMock() + mock_query2.join.return_value.filter.return_value.first.return_value = (mock_account, "normal") + + mock_db_dependencies["db"].session.query.side_effect = [mock_query1, mock_query2] + + # Execute test + result = RegisterService.get_invitation_if_token_valid("tenant-456", "test@example.com", "token-123") + + # Verify results + assert result is None + + # ==================== Helper Method Tests ==================== + + def test_get_invitation_token_key(self): + """Test the _get_invitation_token_key helper method.""" + # Execute test + result = RegisterService._get_invitation_token_key("test-token") + + # Verify results + assert result == "member_invite:token:test-token" + + def test_get_invitation_by_token_with_workspace_and_email(self, mock_redis_dependencies): + """Test _get_invitation_by_token with workspace ID and email.""" + # Setup mock + mock_redis_dependencies.get.return_value = b"user-123" + + # Execute test + result = RegisterService._get_invitation_by_token("token-123", "workspace-456", "test@example.com") + + # Verify results + assert result is not None + assert result["account_id"] == "user-123" + assert result["email"] == "test@example.com" + assert result["workspace_id"] == "workspace-456" + + def test_get_invitation_by_token_without_workspace_and_email(self, mock_redis_dependencies): + """Test _get_invitation_by_token without workspace ID and email.""" + # Setup mock + invitation_data = { + "account_id": "user-123", + "email": "test@example.com", + "workspace_id": "tenant-456", + } + mock_redis_dependencies.get.return_value = json.dumps(invitation_data).encode() + + # Execute test + result = RegisterService._get_invitation_by_token("token-123") + + # Verify results + assert result is not None + assert result == invitation_data + + def test_get_invitation_by_token_no_data(self, mock_redis_dependencies): + """Test _get_invitation_by_token with no data.""" + # Setup mock + mock_redis_dependencies.get.return_value = None + + # Execute test + result = RegisterService._get_invitation_by_token("token-123") + + # Verify results + assert result is None diff --git a/api/tests/unit_tests/services/test_dataset_service_update_dataset.py b/api/tests/unit_tests/services/test_dataset_service_update_dataset.py index cdbb439c85..87b46f213b 100644 --- a/api/tests/unit_tests/services/test_dataset_service_update_dataset.py +++ b/api/tests/unit_tests/services/test_dataset_service_update_dataset.py @@ -10,7 +10,6 @@ from core.model_runtime.entities.model_entities import ModelType from models.dataset import Dataset, ExternalKnowledgeBindings from services.dataset_service import DatasetService from services.errors.account import NoPermissionError -from tests.unit_tests.conftest import redis_mock class DatasetUpdateTestDataFactory: diff --git a/api/tests/unit_tests/services/workflow/test_workflow_deletion.py b/api/tests/unit_tests/services/workflow/test_workflow_deletion.py index 223020c2c5..2c87eaf805 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_deletion.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_deletion.py @@ -10,7 +10,8 @@ from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseE @pytest.fixture def workflow_setup(): - workflow_service = WorkflowService() + mock_session_maker = MagicMock() + workflow_service = WorkflowService(mock_session_maker) session = MagicMock(spec=Session) tenant_id = "test-tenant-id" workflow_id = "test-workflow-id" diff --git a/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py b/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py index c5c9cf1050..8b1348b75b 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py @@ -1,14 +1,14 @@ import dataclasses import secrets -from unittest import mock -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest +from sqlalchemy import Engine from sqlalchemy.orm import Session from core.variables import StringSegment from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID -from core.workflow.nodes import NodeType +from core.workflow.nodes.enums import NodeType from models.enums import DraftVariableType from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel, is_system_variable_editable from services.workflow_draft_variable_service import ( @@ -18,13 +18,25 @@ from services.workflow_draft_variable_service import ( ) +@pytest.fixture +def mock_engine() -> Engine: + return Mock(spec=Engine) + + +@pytest.fixture +def mock_session(mock_engine) -> Session: + mock_session = Mock(spec=Session) + mock_session.get_bind.return_value = mock_engine + return mock_session + + class TestDraftVariableSaver: def _get_test_app_id(self): suffix = secrets.token_hex(6) return f"test_app_id_{suffix}" def test__should_variable_be_visible(self): - mock_session = mock.MagicMock(spec=Session) + mock_session = MagicMock(spec=Session) test_app_id = self._get_test_app_id() saver = DraftVariableSaver( session=mock_session, @@ -70,7 +82,7 @@ class TestDraftVariableSaver: ), ] - mock_session = mock.MagicMock(spec=Session) + mock_session = MagicMock(spec=Session) test_app_id = self._get_test_app_id() saver = DraftVariableSaver( session=mock_session, @@ -105,9 +117,8 @@ class TestWorkflowDraftVariableService: conversation_variables=[], ) - def test_reset_conversation_variable(self): + def test_reset_conversation_variable(self, mock_session): """Test resetting a conversation variable""" - mock_session = Mock(spec=Session) service = WorkflowDraftVariableService(mock_session) test_app_id = self._get_test_app_id() @@ -131,9 +142,8 @@ class TestWorkflowDraftVariableService: mock_reset_conv.assert_called_once_with(workflow, variable) assert result == expected_result - def test_reset_node_variable_with_no_execution_id(self): + def test_reset_node_variable_with_no_execution_id(self, mock_session): """Test resetting a node variable with no execution ID - should delete variable""" - mock_session = Mock(spec=Session) service = WorkflowDraftVariableService(mock_session) test_app_id = self._get_test_app_id() @@ -158,11 +168,26 @@ class TestWorkflowDraftVariableService: mock_session.flush.assert_called_once() assert result is None - def test_reset_node_variable_with_missing_execution_record(self): + def test_reset_node_variable_with_missing_execution_record( + self, + mock_engine, + mock_session, + monkeypatch, + ): """Test resetting a node variable when execution record doesn't exist""" - mock_session = Mock(spec=Session) + mock_repo_session = Mock(spec=Session) + + mock_session_maker = MagicMock() + # Mock the context manager protocol for sessionmaker + mock_session_maker.return_value.__enter__.return_value = mock_repo_session + mock_session_maker.return_value.__exit__.return_value = None + monkeypatch.setattr("services.workflow_draft_variable_service.sessionmaker", mock_session_maker) service = WorkflowDraftVariableService(mock_session) + # Mock the repository to return None (no execution record found) + service._api_node_execution_repo = Mock() + service._api_node_execution_repo.get_execution_by_id.return_value = None + test_app_id = self._get_test_app_id() workflow = self._create_test_workflow(test_app_id) @@ -171,24 +196,41 @@ class TestWorkflowDraftVariableService: variable = WorkflowDraftVariable.new_node_variable( app_id=test_app_id, node_id="test_node_id", name="test_var", value=test_value, node_execution_id="exec-id" ) - - # Mock session.scalars to return None (no execution record found) - mock_scalars = Mock() - mock_scalars.first.return_value = None - mock_session.scalars.return_value = mock_scalars + # Variable is editable by default from factory method result = service._reset_node_var_or_sys_var(workflow, variable) + mock_session_maker.assert_called_once_with(bind=mock_engine, expire_on_commit=False) # Should delete the variable and return None mock_session.delete.assert_called_once_with(instance=variable) mock_session.flush.assert_called_once() assert result is None - def test_reset_node_variable_with_valid_execution_record(self): + def test_reset_node_variable_with_valid_execution_record( + self, + mock_session, + monkeypatch, + ): """Test resetting a node variable with valid execution record - should restore from execution""" - mock_session = Mock(spec=Session) + mock_repo_session = Mock(spec=Session) + + mock_session_maker = MagicMock() + # Mock the context manager protocol for sessionmaker + mock_session_maker.return_value.__enter__.return_value = mock_repo_session + mock_session_maker.return_value.__exit__.return_value = None + mock_session_maker = monkeypatch.setattr( + "services.workflow_draft_variable_service.sessionmaker", mock_session_maker + ) service = WorkflowDraftVariableService(mock_session) + # Create mock execution record + mock_execution = Mock(spec=WorkflowNodeExecutionModel) + mock_execution.outputs_dict = {"test_var": "output_value"} + + # Mock the repository to return the execution record + service._api_node_execution_repo = Mock() + service._api_node_execution_repo.get_execution_by_id.return_value = mock_execution + test_app_id = self._get_test_app_id() workflow = self._create_test_workflow(test_app_id) @@ -197,16 +239,7 @@ class TestWorkflowDraftVariableService: variable = WorkflowDraftVariable.new_node_variable( app_id=test_app_id, node_id="test_node_id", name="test_var", value=test_value, node_execution_id="exec-id" ) - - # Create mock execution record - mock_execution = Mock(spec=WorkflowNodeExecutionModel) - mock_execution.process_data_dict = {"test_var": "process_value"} - mock_execution.outputs_dict = {"test_var": "output_value"} - - # Mock session.scalars to return the execution record - mock_scalars = Mock() - mock_scalars.first.return_value = mock_execution - mock_session.scalars.return_value = mock_scalars + # Variable is editable by default from factory method # Mock workflow methods mock_node_config = {"type": "test_node"} @@ -224,9 +257,8 @@ class TestWorkflowDraftVariableService: # Should return the updated variable assert result == variable - def test_reset_non_editable_system_variable_raises_error(self): + def test_reset_non_editable_system_variable_raises_error(self, mock_session): """Test that resetting a non-editable system variable raises an error""" - mock_session = Mock(spec=Session) service = WorkflowDraftVariableService(mock_session) test_app_id = self._get_test_app_id() @@ -242,24 +274,13 @@ class TestWorkflowDraftVariableService: editable=False, # Non-editable system variable ) - # Mock the service to properly check system variable editability - with patch.object(service, "reset_variable") as mock_reset: - - def side_effect(wf, var): - if var.get_variable_type() == DraftVariableType.SYS and not is_system_variable_editable(var.name): - raise VariableResetError(f"cannot reset system variable, variable_id={var.id}") - return var - - mock_reset.side_effect = side_effect - - with pytest.raises(VariableResetError) as exc_info: - service.reset_variable(workflow, variable) - assert "cannot reset system variable" in str(exc_info.value) - assert f"variable_id={variable.id}" in str(exc_info.value) + with pytest.raises(VariableResetError) as exc_info: + service.reset_variable(workflow, variable) + assert "cannot reset system variable" in str(exc_info.value) + assert f"variable_id={variable.id}" in str(exc_info.value) - def test_reset_editable_system_variable_succeeds(self): + def test_reset_editable_system_variable_succeeds(self, mock_session): """Test that resetting an editable system variable succeeds""" - mock_session = Mock(spec=Session) service = WorkflowDraftVariableService(mock_session) test_app_id = self._get_test_app_id() @@ -279,10 +300,9 @@ class TestWorkflowDraftVariableService: mock_execution = Mock(spec=WorkflowNodeExecutionModel) mock_execution.outputs_dict = {"sys.files": "[]"} - # Mock session.scalars to return the execution record - mock_scalars = Mock() - mock_scalars.first.return_value = mock_execution - mock_session.scalars.return_value = mock_scalars + # Mock the repository to return the execution record + service._api_node_execution_repo = Mock() + service._api_node_execution_repo.get_execution_by_id.return_value = mock_execution result = service._reset_node_var_or_sys_var(workflow, variable) @@ -291,9 +311,8 @@ class TestWorkflowDraftVariableService: assert variable.last_edited_at is None mock_session.flush.assert_called() - def test_reset_query_system_variable_succeeds(self): + def test_reset_query_system_variable_succeeds(self, mock_session): """Test that resetting query system variable (another editable one) succeeds""" - mock_session = Mock(spec=Session) service = WorkflowDraftVariableService(mock_session) test_app_id = self._get_test_app_id() @@ -313,10 +332,9 @@ class TestWorkflowDraftVariableService: mock_execution = Mock(spec=WorkflowNodeExecutionModel) mock_execution.outputs_dict = {"sys.query": "reset query"} - # Mock session.scalars to return the execution record - mock_scalars = Mock() - mock_scalars.first.return_value = mock_execution - mock_session.scalars.return_value = mock_scalars + # Mock the repository to return the execution record + service._api_node_execution_repo = Mock() + service._api_node_execution_repo.get_execution_by_id.return_value = mock_execution result = service._reset_node_var_or_sys_var(workflow, variable) diff --git a/api/tests/unit_tests/services/workflow/test_workflow_node_execution_service_repository.py b/api/tests/unit_tests/services/workflow/test_workflow_node_execution_service_repository.py new file mode 100644 index 0000000000..32d2f8b7e0 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_workflow_node_execution_service_repository.py @@ -0,0 +1,288 @@ +from datetime import datetime +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest +from sqlalchemy.orm import Session + +from models.workflow import WorkflowNodeExecutionModel +from repositories.sqlalchemy_api_workflow_node_execution_repository import ( + DifyAPISQLAlchemyWorkflowNodeExecutionRepository, +) + + +class TestSQLAlchemyWorkflowNodeExecutionServiceRepository: + @pytest.fixture + def repository(self): + mock_session_maker = MagicMock() + return DifyAPISQLAlchemyWorkflowNodeExecutionRepository(session_maker=mock_session_maker) + + @pytest.fixture + def mock_execution(self): + execution = MagicMock(spec=WorkflowNodeExecutionModel) + execution.id = str(uuid4()) + execution.tenant_id = "tenant-123" + execution.app_id = "app-456" + execution.workflow_id = "workflow-789" + execution.workflow_run_id = "run-101" + execution.node_id = "node-202" + execution.index = 1 + execution.created_at = "2023-01-01T00:00:00Z" + return execution + + def test_get_node_last_execution_found(self, repository, mock_execution): + """Test getting the last execution for a node when it exists.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + mock_session.scalar.return_value = mock_execution + + # Act + result = repository.get_node_last_execution( + tenant_id="tenant-123", + app_id="app-456", + workflow_id="workflow-789", + node_id="node-202", + ) + + # Assert + assert result == mock_execution + mock_session.scalar.assert_called_once() + # Verify the query was constructed correctly + call_args = mock_session.scalar.call_args[0][0] + assert hasattr(call_args, "compile") # It's a SQLAlchemy statement + + def test_get_node_last_execution_not_found(self, repository): + """Test getting the last execution for a node when it doesn't exist.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + mock_session.scalar.return_value = None + + # Act + result = repository.get_node_last_execution( + tenant_id="tenant-123", + app_id="app-456", + workflow_id="workflow-789", + node_id="node-202", + ) + + # Assert + assert result is None + mock_session.scalar.assert_called_once() + + def test_get_executions_by_workflow_run(self, repository, mock_execution): + """Test getting all executions for a workflow run.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + executions = [mock_execution] + mock_session.execute.return_value.scalars.return_value.all.return_value = executions + + # Act + result = repository.get_executions_by_workflow_run( + tenant_id="tenant-123", + app_id="app-456", + workflow_run_id="run-101", + ) + + # Assert + assert result == executions + mock_session.execute.assert_called_once() + # Verify the query was constructed correctly + call_args = mock_session.execute.call_args[0][0] + assert hasattr(call_args, "compile") # It's a SQLAlchemy statement + + def test_get_executions_by_workflow_run_empty(self, repository): + """Test getting executions for a workflow run when none exist.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + mock_session.execute.return_value.scalars.return_value.all.return_value = [] + + # Act + result = repository.get_executions_by_workflow_run( + tenant_id="tenant-123", + app_id="app-456", + workflow_run_id="run-101", + ) + + # Assert + assert result == [] + mock_session.execute.assert_called_once() + + def test_get_execution_by_id_found(self, repository, mock_execution): + """Test getting execution by ID when it exists.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + mock_session.scalar.return_value = mock_execution + + # Act + result = repository.get_execution_by_id(mock_execution.id) + + # Assert + assert result == mock_execution + mock_session.scalar.assert_called_once() + + def test_get_execution_by_id_not_found(self, repository): + """Test getting execution by ID when it doesn't exist.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + mock_session.scalar.return_value = None + + # Act + result = repository.get_execution_by_id("non-existent-id") + + # Assert + assert result is None + mock_session.scalar.assert_called_once() + + def test_repository_implements_protocol(self, repository): + """Test that the repository implements the required protocol methods.""" + # Verify all protocol methods are implemented + assert hasattr(repository, "get_node_last_execution") + assert hasattr(repository, "get_executions_by_workflow_run") + assert hasattr(repository, "get_execution_by_id") + + # Verify methods are callable + assert callable(repository.get_node_last_execution) + assert callable(repository.get_executions_by_workflow_run) + assert callable(repository.get_execution_by_id) + assert callable(repository.delete_expired_executions) + assert callable(repository.delete_executions_by_app) + assert callable(repository.get_expired_executions_batch) + assert callable(repository.delete_executions_by_ids) + + def test_delete_expired_executions(self, repository): + """Test deleting expired executions.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + + # Mock the select query to return some IDs first time, then empty to stop loop + execution_ids = ["id1", "id2"] # Less than batch_size to trigger break + + # Mock execute method to handle both select and delete statements + def mock_execute(stmt): + mock_result = MagicMock() + # For select statements, return execution IDs + if hasattr(stmt, "limit"): # This is our select statement + mock_result.scalars.return_value.all.return_value = execution_ids + else: # This is our delete statement + mock_result.rowcount = 2 + return mock_result + + mock_session.execute.side_effect = mock_execute + + before_date = datetime(2023, 1, 1) + + # Act + result = repository.delete_expired_executions( + tenant_id="tenant-123", + before_date=before_date, + batch_size=1000, + ) + + # Assert + assert result == 2 + assert mock_session.execute.call_count == 2 # One select call, one delete call + mock_session.commit.assert_called_once() + + def test_delete_executions_by_app(self, repository): + """Test deleting executions by app.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + + # Mock the select query to return some IDs first time, then empty to stop loop + execution_ids = ["id1", "id2"] + + # Mock execute method to handle both select and delete statements + def mock_execute(stmt): + mock_result = MagicMock() + # For select statements, return execution IDs + if hasattr(stmt, "limit"): # This is our select statement + mock_result.scalars.return_value.all.return_value = execution_ids + else: # This is our delete statement + mock_result.rowcount = 2 + return mock_result + + mock_session.execute.side_effect = mock_execute + + # Act + result = repository.delete_executions_by_app( + tenant_id="tenant-123", + app_id="app-456", + batch_size=1000, + ) + + # Assert + assert result == 2 + assert mock_session.execute.call_count == 2 # One select call, one delete call + mock_session.commit.assert_called_once() + + def test_get_expired_executions_batch(self, repository): + """Test getting expired executions batch for backup.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + + # Create mock execution objects + mock_execution1 = MagicMock() + mock_execution1.id = "exec-1" + mock_execution2 = MagicMock() + mock_execution2.id = "exec-2" + + mock_session.execute.return_value.scalars.return_value.all.return_value = [mock_execution1, mock_execution2] + + before_date = datetime(2023, 1, 1) + + # Act + result = repository.get_expired_executions_batch( + tenant_id="tenant-123", + before_date=before_date, + batch_size=1000, + ) + + # Assert + assert len(result) == 2 + assert result[0].id == "exec-1" + assert result[1].id == "exec-2" + mock_session.execute.assert_called_once() + + def test_delete_executions_by_ids(self, repository): + """Test deleting executions by IDs.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + + # Mock the delete query result + mock_result = MagicMock() + mock_result.rowcount = 3 + mock_session.execute.return_value = mock_result + + execution_ids = ["id1", "id2", "id3"] + + # Act + result = repository.delete_executions_by_ids(execution_ids) + + # Assert + assert result == 3 + mock_session.execute.assert_called_once() + mock_session.commit.assert_called_once() + + def test_delete_executions_by_ids_empty_list(self, repository): + """Test deleting executions with empty ID list.""" + # Arrange + mock_session = MagicMock(spec=Session) + repository._session_maker.return_value.__enter__.return_value = mock_session + + # Act + result = repository.delete_executions_by_ids([]) + + # Assert + assert result == 0 + mock_session.query.assert_not_called() + mock_session.commit.assert_not_called() diff --git a/api/tests/unit_tests/services/workflow/test_workflow_service.py b/api/tests/unit_tests/services/workflow/test_workflow_service.py index 13393668ea..9700cbaf0e 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_service.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_service.py @@ -10,7 +10,8 @@ from services.workflow_service import WorkflowService class TestWorkflowService: @pytest.fixture def workflow_service(self): - return WorkflowService() + mock_session_maker = MagicMock() + return WorkflowService(mock_session_maker) @pytest.fixture def mock_app(self): diff --git a/api/tests/unit_tests/utils/structured_output_parser/test_structured_output_parser.py b/api/tests/unit_tests/utils/structured_output_parser/test_structured_output_parser.py index 728c58fc5b..93284eed4b 100644 --- a/api/tests/unit_tests/utils/structured_output_parser/test_structured_output_parser.py +++ b/api/tests/unit_tests/utils/structured_output_parser/test_structured_output_parser.py @@ -27,11 +27,11 @@ def create_mock_usage(prompt_tokens: int = 10, completion_tokens: int = 5) -> LL return LLMUsage( prompt_tokens=prompt_tokens, prompt_unit_price=Decimal("0.001"), - prompt_price_unit=Decimal("1"), + prompt_price_unit=Decimal(1), prompt_price=Decimal(str(prompt_tokens)) * Decimal("0.001"), completion_tokens=completion_tokens, completion_unit_price=Decimal("0.002"), - completion_price_unit=Decimal("1"), + completion_price_unit=Decimal(1), completion_price=Decimal(str(completion_tokens)) * Decimal("0.002"), total_tokens=prompt_tokens + completion_tokens, total_price=Decimal(str(prompt_tokens)) * Decimal("0.001") + Decimal(str(completion_tokens)) * Decimal("0.002"), diff --git a/api/uv.lock b/api/uv.lock index d379f28e52..21b6b20f53 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -99,28 +99,29 @@ wheels = [ [[package]] name = "aiosignal" -version = "1.3.2" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/b5/6d55e80f6d8a08ce22b982eafa278d823b541c925f11ee774b0b9c43473d/aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54", size = 19424, upload-time = "2024-12-13T17:10:40.86Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597, upload-time = "2024-12-13T17:10:38.469Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] [[package]] name = "alembic" -version = "1.16.2" +version = "1.16.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9c/35/116797ff14635e496bbda0c168987f5326a6555b09312e9b817e360d1f56/alembic-1.16.2.tar.gz", hash = "sha256:e53c38ff88dadb92eb22f8b150708367db731d58ad7e9d417c9168ab516cbed8", size = 1963563, upload-time = "2025-06-16T18:05:08.566Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/40/28683414cc8711035a65256ca689e159471aa9ef08e8741ad1605bc01066/alembic-1.16.3.tar.gz", hash = "sha256:18ad13c1f40a5796deee4b2346d1a9c382f44b8af98053897484fa6cf88025e4", size = 1967462, upload-time = "2025-07-08T18:57:50.991Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/e2/88e425adac5ad887a087c38d04fe2030010572a3e0e627f8a6e8c33eeda8/alembic-1.16.2-py3-none-any.whl", hash = "sha256:5f42e9bd0afdbd1d5e3ad856c01754530367debdebf21ed6894e34af52b3bb03", size = 242717, upload-time = "2025-06-16T18:05:10.27Z" }, + { url = "https://files.pythonhosted.org/packages/e6/68/1dea77887af7304528ea944c355d769a7ccc4599d3a23bd39182486deb42/alembic-1.16.3-py3-none-any.whl", hash = "sha256:70a7c7829b792de52d08ca0e3aefaf060687cb8ed6bebfa557e597a1a5e5a481", size = 246933, upload-time = "2025-07-08T18:57:52.793Z" }, ] [[package]] @@ -243,7 +244,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/22/8a/ef8ddf5ee0350984c [[package]] name = "alibabacloud-tea-openapi" -version = "0.3.15" +version = "0.3.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alibabacloud-credentials" }, @@ -252,7 +253,7 @@ dependencies = [ { name = "alibabacloud-tea-util" }, { name = "alibabacloud-tea-xml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/be/cb/f1b10b1da37e4c0de2aa9ca1e7153a6960a7f2dc496664e85fdc8b621f84/alibabacloud_tea_openapi-0.3.15.tar.gz", hash = "sha256:56a0aa6d51d8cf18c0cf3d219d861f4697f59d3e17fa6726b1101826d93988a2", size = 13021, upload-time = "2025-05-06T12:56:29.402Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/be/f594e79625e5ccfcfe7f12d7d70709a3c59e920878469c998886211c850d/alibabacloud_tea_openapi-0.3.16.tar.gz", hash = "sha256:6bffed8278597592e67860156f424bde4173a6599d7b6039fb640a3612bae292", size = 13087, upload-time = "2025-07-04T09:30:10.689Z" } [[package]] name = "alibabacloud-tea-util" @@ -370,11 +371,11 @@ wheels = [ [[package]] name = "asgiref" -version = "3.8.1" +version = "3.9.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/29/38/b3395cc9ad1b56d2ddac9970bc8f4141312dbaec28bc7c218b0dfafd0f42/asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590", size = 35186, upload-time = "2024-03-22T14:39:36.863Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/61/0aa957eec22ff70b830b22ff91f825e70e1ef732c06666a805730f28b36b/asgiref-3.9.1.tar.gz", hash = "sha256:a5ab6582236218e5ef1648f242fd9f10626cfd4de8dc377db215d5d5098e3142", size = 36870, upload-time = "2025-07-08T09:07:43.344Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", size = 23828, upload-time = "2024-03-22T14:39:34.521Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3c/0464dcada90d5da0e71018c04a140ad6349558afb30b3051b4264cc5b965/asgiref-3.9.1-py3-none-any.whl", hash = "sha256:f3bba7092a48005b5f5bacd747d36ee4a5a61f4a269a6df590b43144355ebd2c", size = 23790, upload-time = "2025-07-08T09:07:41.548Z" }, ] [[package]] @@ -559,16 +560,16 @@ wheels = [ [[package]] name = "boto3-stubs" -version = "1.39.2" +version = "1.39.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore-stubs" }, { name = "types-s3transfer" }, { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/09/206a17938bfc7ec6e7c0b13ed58ad78146e46c29436d324ed55ceb5136ed/boto3_stubs-1.39.2.tar.gz", hash = "sha256:b1f1baef1658bd575a29ca85cc0877dbb3adeb376ffa8cbf242b876719ae0f95", size = 99939, upload-time = "2025-07-02T19:28:20.423Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/ea/85b9940d6eedc04d0c6febf24d27311b6ee54f85ccc37192eb4db0dff5d6/boto3_stubs-1.39.3.tar.gz", hash = "sha256:9aad443b1d690951fd9ccb6fa20ad387bd0b1054c704566ff65dd0043a63fc26", size = 99947, upload-time = "2025-07-03T19:28:15.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/be/9c65f2bfc6df27ec5f16d28c454e2e3cb9a7af3ef8588440658334325a85/boto3_stubs-1.39.2-py3-none-any.whl", hash = "sha256:ce98d96fe1a7177b05067be3cd933277c88f745de836752f9ef8b4286dbfa53b", size = 69196, upload-time = "2025-07-02T19:28:07.025Z" }, + { url = "https://files.pythonhosted.org/packages/be/b8/0c56297e5f290de17e838c7e4ff338f5b94351c6566aed70ee197a671dc5/boto3_stubs-1.39.3-py3-none-any.whl", hash = "sha256:4daddb19374efa6d1bef7aded9cede0075f380722a9e60ab129ebba14ae66b69", size = 69196, upload-time = "2025-07-03T19:28:09.4Z" }, ] [package.optional-dependencies] @@ -1216,7 +1217,7 @@ wheels = [ [[package]] name = "dify-api" -version = "1.5.1" +version = "1.6.0" source = { virtual = "." } dependencies = [ { name = "arize-phoenix-otel" }, @@ -1245,6 +1246,7 @@ dependencies = [ { name = "googleapis-common-protos" }, { name = "gunicorn" }, { name = "httpx", extra = ["socks"] }, + { name = "httpx-sse" }, { name = "jieba" }, { name = "json-repair" }, { name = "langfuse" }, @@ -1289,6 +1291,7 @@ dependencies = [ { name = "sendgrid" }, { name = "sentry-sdk", extra = ["flask"] }, { name = "sqlalchemy" }, + { name = "sseclient-py" }, { name = "starlette" }, { name = "tiktoken" }, { name = "transformers" }, @@ -1425,6 +1428,7 @@ requires-dist = [ { name = "googleapis-common-protos", specifier = "==1.63.0" }, { name = "gunicorn", specifier = "~=23.0.0" }, { name = "httpx", extras = ["socks"], specifier = "~=0.27.0" }, + { name = "httpx-sse", specifier = ">=0.4.0" }, { name = "jieba", specifier = "==0.42.1" }, { name = "json-repair", specifier = ">=0.41.1" }, { name = "langfuse", specifier = "~=2.51.3" }, @@ -1469,6 +1473,7 @@ requires-dist = [ { name = "sendgrid", specifier = "~=6.12.3" }, { name = "sentry-sdk", extras = ["flask"], specifier = "~=2.28.0" }, { name = "sqlalchemy", specifier = "~=2.0.29" }, + { name = "sseclient-py", specifier = ">=1.8.0" }, { name = "starlette", specifier = "==0.41.0" }, { name = "tiktoken", specifier = "~=0.9.0" }, { name = "transformers", specifier = "~=4.51.0" }, @@ -1493,7 +1498,7 @@ dev = [ { name = "pytest-cov", specifier = "~=4.1.0" }, { name = "pytest-env", specifier = "~=1.1.3" }, { name = "pytest-mock", specifier = "~=3.14.0" }, - { name = "ruff", specifier = "~=0.11.5" }, + { name = "ruff", specifier = "~=0.12.3" }, { name = "scipy-stubs", specifier = ">=1.15.3.0" }, { name = "types-aiofiles", specifier = "~=24.1.0" }, { name = "types-beautifulsoup4", specifier = "~=4.12.0" }, @@ -1708,16 +1713,16 @@ wheels = [ [[package]] name = "fastapi" -version = "0.115.14" +version = "0.116.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "starlette" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ca/53/8c38a874844a8b0fa10dd8adf3836ac154082cf88d3f22b544e9ceea0a15/fastapi-0.115.14.tar.gz", hash = "sha256:b1de15cdc1c499a4da47914db35d0e4ef8f1ce62b624e94e0e5824421df99739", size = 296263, upload-time = "2025-06-26T15:29:08.21Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/38/e1da78736143fd885c36213a3ccc493c384ae8fea6a0f0bc272ef42ebea8/fastapi-0.116.0.tar.gz", hash = "sha256:80dc0794627af0390353a6d1171618276616310d37d24faba6648398e57d687a", size = 296518, upload-time = "2025-07-07T15:09:27.82Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/50/b1222562c6d270fea83e9c9075b8e8600b8479150a18e4516a6138b980d1/fastapi-0.115.14-py3-none-any.whl", hash = "sha256:6c0c8bf9420bd58f565e585036d971872472b4f7d3f6c73b698e10cffdefb3ca", size = 95514, upload-time = "2025-06-26T15:29:06.49Z" }, + { url = "https://files.pythonhosted.org/packages/2f/68/d80347fe2360445b5f58cf290e588a4729746e7501080947e6cdae114b1f/fastapi-0.116.0-py3-none-any.whl", hash = "sha256:fdcc9ed272eaef038952923bef2b735c02372402d1203ee1210af4eea7a78d2b", size = 95625, upload-time = "2025-07-07T15:09:26.348Z" }, ] [[package]] @@ -2532,6 +2537,15 @@ socks = [ { name = "socksio" }, ] +[[package]] +name = "httpx-sse" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/fa/66bd985dd0b7c109a3bcb89272ee0bfb7e2b4d06309ad7b38ff866734b2a/httpx_sse-0.4.1.tar.gz", hash = "sha256:8f44d34414bc7b21bf3602713005c5df4917884f76072479b21f68befa4ea26e", size = 12998, upload-time = "2025-06-24T13:21:05.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/0a/6269e3473b09aed2dab8aa1a600c70f31f00ae1349bee30658f7e358a159/httpx_sse-0.4.1-py3-none-any.whl", hash = "sha256:cba42174344c3a5b06f255ce65b350880f962d99ead85e776f23c6618a377a37", size = 8054, upload-time = "2025-06-24T13:21:04.772Z" }, +] + [[package]] name = "huggingface-hub" version = "0.33.2" @@ -2574,15 +2588,15 @@ wheels = [ [[package]] name = "hypothesis" -version = "6.135.24" +version = "6.135.26" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "sortedcontainers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/ae/f846b67ce9fc80cf51cece6b7adaa3fe2de4251242d142e241ce5d4aa26f/hypothesis-6.135.24.tar.gz", hash = "sha256:e301aeb2691ec0a1f62bfc405eaa966055d603e328cd854c1ed59e1728e35ab6", size = 454011, upload-time = "2025-07-03T02:46:51.776Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/83/15c4e30561a0d8c8d076c88cb159187823d877118f34c851ada3b9b02a7b/hypothesis-6.135.26.tar.gz", hash = "sha256:73af0e46cd5039c6806f514fed6a3c185d91ef88b5a1577477099ddbd1a2e300", size = 454523, upload-time = "2025-07-05T04:59:45.443Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/cb/c38acf27826a96712302229622f32dd356b9c4fbe52a3e9f615706027af8/hypothesis-6.135.24-py3-none-any.whl", hash = "sha256:88ed21fbfa481ca9851a9080841b3caca14cd4ed51a165dfae8006325775ee72", size = 520920, upload-time = "2025-07-03T02:46:48.286Z" }, + { url = "https://files.pythonhosted.org/packages/3c/78/db4fdc464219455f8dde90074660c3faf8429101b2d1299cac7d219e3176/hypothesis-6.135.26-py3-none-any.whl", hash = "sha256:fa237cbe2ae2c31d65f7230dcb866139ace635dcfec6c30dddf25974dd8ff4b9", size = 521517, upload-time = "2025-07-05T04:59:42.061Z" }, ] [[package]] @@ -2892,10 +2906,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/55/2cb24ea48aa30c99f805921c1c7860c1f45c0e811e44ee4e6a155668de06/lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563", size = 4952289, upload-time = "2025-06-28T18:47:25.602Z" }, { url = "https://files.pythonhosted.org/packages/31/c0/b25d9528df296b9a3306ba21ff982fc5b698c45ab78b94d18c2d6ae71fd9/lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7", size = 5111310, upload-time = "2025-06-28T18:47:28.136Z" }, { url = "https://files.pythonhosted.org/packages/e9/af/681a8b3e4f668bea6e6514cbcb297beb6de2b641e70f09d3d78655f4f44c/lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7", size = 5025457, upload-time = "2025-06-26T16:26:15.068Z" }, + { url = "https://files.pythonhosted.org/packages/99/b6/3a7971aa05b7be7dfebc7ab57262ec527775c2c3c5b2f43675cac0458cad/lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991", size = 5657016, upload-time = "2025-07-03T19:19:06.008Z" }, { url = "https://files.pythonhosted.org/packages/69/f8/693b1a10a891197143c0673fcce5b75fc69132afa81a36e4568c12c8faba/lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da", size = 5257565, upload-time = "2025-06-26T16:26:17.906Z" }, { url = "https://files.pythonhosted.org/packages/a8/96/e08ff98f2c6426c98c8964513c5dab8d6eb81dadcd0af6f0c538ada78d33/lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e", size = 4713390, upload-time = "2025-06-26T16:26:20.292Z" }, { url = "https://files.pythonhosted.org/packages/a8/83/6184aba6cc94d7413959f6f8f54807dc318fdcd4985c347fe3ea6937f772/lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741", size = 5066103, upload-time = "2025-06-26T16:26:22.765Z" }, { url = "https://files.pythonhosted.org/packages/ee/01/8bf1f4035852d0ff2e36a4d9aacdbcc57e93a6cd35a54e05fa984cdf73ab/lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3", size = 4791428, upload-time = "2025-06-26T16:26:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/29/31/c0267d03b16954a85ed6b065116b621d37f559553d9339c7dcc4943a76f1/lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16", size = 5678523, upload-time = "2025-07-03T19:19:09.837Z" }, { url = "https://files.pythonhosted.org/packages/5c/f7/5495829a864bc5f8b0798d2b52a807c89966523140f3d6fa3a58ab6720ea/lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0", size = 5281290, upload-time = "2025-06-26T16:26:29.406Z" }, { url = "https://files.pythonhosted.org/packages/79/56/6b8edb79d9ed294ccc4e881f4db1023af56ba451909b9ce79f2a2cd7c532/lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a", size = 3613495, upload-time = "2025-06-26T16:26:31.588Z" }, { url = "https://files.pythonhosted.org/packages/0b/1e/cc32034b40ad6af80b6fd9b66301fc0f180f300002e5c3eb5a6110a93317/lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3", size = 4014711, upload-time = "2025-06-26T16:26:33.723Z" }, @@ -3732,7 +3748,7 @@ wheels = [ [[package]] name = "opik" -version = "1.7.41" +version = "1.7.43" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3-stubs", extra = ["bedrock-runtime"] }, @@ -3751,9 +3767,9 @@ dependencies = [ { name = "tqdm" }, { name = "uuid6" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/81/6cddb705b3f416cfe4f0507916f51d0886087695f9dab49cfc6b00eb0266/opik-1.7.41.tar.gz", hash = "sha256:6ce2f72c7d23a62e2c13d419ce50754f6e17234825dcf26506e7def34dd38e26", size = 323333, upload-time = "2025-07-02T12:35:31.76Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/52/cea0317bc3207bc967b48932781995d9cdb2c490e7e05caa00ff660f7205/opik-1.7.43.tar.gz", hash = "sha256:0b02522b0b74d0a67b141939deda01f8bb69690eda6b04a7cecb1c7f0649ccd0", size = 326886, upload-time = "2025-07-07T10:30:07.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/46/ee27d06cc2049619806c992bdaa10e25b93d19ecedbc5c0fa772d8ac9a6d/opik-1.7.41-py3-none-any.whl", hash = "sha256:99df9c7b7b504777a51300b27a72bc646903201629611082b9b1f3c3adfbb3bf", size = 614890, upload-time = "2025-07-02T12:35:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/76/ae/f3566bdc3c49a1a8f795b1b6e726ef211c87e31f92d870ca6d63999c9bbf/opik-1.7.43-py3-none-any.whl", hash = "sha256:a66395c8b5ea7c24846f72dafc70c74d5b8f24ffbc4c8a1b3a7f9456e550568d", size = 625356, upload-time = "2025-07-07T10:30:06.389Z" }, ] [[package]] @@ -3975,6 +3991,8 @@ sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3 wheels = [ { url = "https://files.pythonhosted.org/packages/db/26/77f8ed17ca4ffd60e1dcd220a6ec6d71210ba398cfa33a13a1cd614c5613/pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722", size = 5316531, upload-time = "2025-07-01T09:13:59.203Z" }, { url = "https://files.pythonhosted.org/packages/cb/39/ee475903197ce709322a17a866892efb560f57900d9af2e55f86db51b0a5/pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288", size = 4686560, upload-time = "2025-07-01T09:14:01.101Z" }, + { url = "https://files.pythonhosted.org/packages/d5/90/442068a160fd179938ba55ec8c97050a612426fae5ec0a764e345839f76d/pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d", size = 5870978, upload-time = "2025-07-03T13:09:55.638Z" }, + { url = "https://files.pythonhosted.org/packages/13/92/dcdd147ab02daf405387f0218dcf792dc6dd5b14d2573d40b4caeef01059/pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494", size = 7641168, upload-time = "2025-07-03T13:10:00.37Z" }, { url = "https://files.pythonhosted.org/packages/6e/db/839d6ba7fd38b51af641aa904e2960e7a5644d60ec754c046b7d2aee00e5/pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58", size = 5973053, upload-time = "2025-07-01T09:14:04.491Z" }, { url = "https://files.pythonhosted.org/packages/f2/2f/d7675ecae6c43e9f12aa8d58b6012683b20b6edfbdac7abcb4e6af7a3784/pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f", size = 6640273, upload-time = "2025-07-01T09:14:06.235Z" }, { url = "https://files.pythonhosted.org/packages/45/ad/931694675ede172e15b2ff03c8144a0ddaea1d87adb72bb07655eaffb654/pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e", size = 6082043, upload-time = "2025-07-01T09:14:07.978Z" }, @@ -3984,6 +4002,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/df/90bd886fabd544c25addd63e5ca6932c86f2b701d5da6c7839387a076b4a/pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd", size = 2423079, upload-time = "2025-07-01T09:14:15.268Z" }, { url = "https://files.pythonhosted.org/packages/40/fe/1bc9b3ee13f68487a99ac9529968035cca2f0a51ec36892060edcc51d06a/pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4", size = 5278800, upload-time = "2025-07-01T09:14:17.648Z" }, { url = "https://files.pythonhosted.org/packages/2c/32/7e2ac19b5713657384cec55f89065fb306b06af008cfd87e572035b27119/pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69", size = 4686296, upload-time = "2025-07-01T09:14:19.828Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1e/b9e12bbe6e4c2220effebc09ea0923a07a6da1e1f1bfbc8d7d29a01ce32b/pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d", size = 5871726, upload-time = "2025-07-03T13:10:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/8d/33/e9200d2bd7ba00dc3ddb78df1198a6e80d7669cce6c2bdbeb2530a74ec58/pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6", size = 7644652, upload-time = "2025-07-03T13:10:10.391Z" }, { url = "https://files.pythonhosted.org/packages/41/f1/6f2427a26fc683e00d985bc391bdd76d8dd4e92fac33d841127eb8fb2313/pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7", size = 5977787, upload-time = "2025-07-01T09:14:21.63Z" }, { url = "https://files.pythonhosted.org/packages/e4/c9/06dd4a38974e24f932ff5f98ea3c546ce3f8c995d3f0985f8e5ba48bba19/pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024", size = 6645236, upload-time = "2025-07-01T09:14:23.321Z" }, { url = "https://files.pythonhosted.org/packages/40/e7/848f69fb79843b3d91241bad658e9c14f39a32f71a301bcd1d139416d1be/pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809", size = 6086950, upload-time = "2025-07-01T09:14:25.237Z" }, @@ -3993,6 +4013,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/8f/b13447d1bf0b1f7467ce7d86f6e6edf66c0ad7cf44cf5c87a37f9bed9936/pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542", size = 2423067, upload-time = "2025-07-01T09:14:33.709Z" }, { url = "https://files.pythonhosted.org/packages/9e/e3/6fa84033758276fb31da12e5fb66ad747ae83b93c67af17f8c6ff4cc8f34/pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6", size = 5270566, upload-time = "2025-07-01T09:16:19.801Z" }, { url = "https://files.pythonhosted.org/packages/5b/ee/e8d2e1ab4892970b561e1ba96cbd59c0d28cf66737fc44abb2aec3795a4e/pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438", size = 4654618, upload-time = "2025-07-01T09:16:21.818Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6d/17f80f4e1f0761f02160fc433abd4109fa1548dcfdca46cfdadaf9efa565/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3", size = 4874248, upload-time = "2025-07-03T13:11:20.738Z" }, + { url = "https://files.pythonhosted.org/packages/de/5f/c22340acd61cef960130585bbe2120e2fd8434c214802f07e8c03596b17e/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c", size = 6583963, upload-time = "2025-07-03T13:11:26.283Z" }, { url = "https://files.pythonhosted.org/packages/31/5e/03966aedfbfcbb4d5f8aa042452d3361f325b963ebbadddac05b122e47dd/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361", size = 4957170, upload-time = "2025-07-01T09:16:23.762Z" }, { url = "https://files.pythonhosted.org/packages/cc/2d/e082982aacc927fc2cab48e1e731bdb1643a1406acace8bed0900a61464e/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7", size = 5581505, upload-time = "2025-07-01T09:16:25.593Z" }, { url = "https://files.pythonhosted.org/packages/34/e7/ae39f538fd6844e982063c3a5e4598b8ced43b9633baa3a85ef33af8c05c/pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8", size = 6984598, upload-time = "2025-07-01T09:16:27.732Z" }, @@ -4065,7 +4087,7 @@ wheels = [ [[package]] name = "posthog" -version = "6.0.2" +version = "6.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -4075,9 +4097,9 @@ dependencies = [ { name = "six" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d9/10/37ea988b3ae73cbfd1f2d5e523cca31cecfcc40cbd0de6511f40462fdb78/posthog-6.0.2.tar.gz", hash = "sha256:94a28e65d7a2d1b2952e53a1b97fa4d6504b8d7e4c197c57f653621e55b549eb", size = 88141, upload-time = "2025-07-02T19:21:50.306Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/a2/1b68562124b0d0e615fa8431cc88c84b3db6526275c2c19a419579a49277/posthog-6.0.3.tar.gz", hash = "sha256:9005abb341af8fedd9d82ca0359b3d35a9537555cdc9881bfb469f7c0b4b0ec5", size = 91861, upload-time = "2025-07-07T07:14:08.21Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/85/2c/0c5dbbf9bc30401ae2a1b6b52b8abc19e4060cf28c3288ae9d962e65e3ad/posthog-6.0.2-py3-none-any.whl", hash = "sha256:756cc9adad9e42961454f8ac391b92a2f70ebb6607d29b0c568de08e5d8f1b18", size = 104946, upload-time = "2025-07-02T19:21:48.77Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f1/a8d86245d41c8686f7d828a4959bdf483e8ac331b249b48b8c61fc884a1c/posthog-6.0.3-py3-none-any.whl", hash = "sha256:4b808c907f3623216a9362d91fdafce8e2f57a8387fb3020475c62ec809be56d", size = 108978, upload-time = "2025-07-07T07:14:06.451Z" }, ] [[package]] @@ -4585,39 +4607,39 @@ wheels = [ [[package]] name = "python-calamine" -version = "0.3.2" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6b/21/387b92059909e741af7837194d84250335d2a057f614752b6364aaaa2f56/python_calamine-0.3.2.tar.gz", hash = "sha256:5cf12f2086373047cdea681711857b672cba77a34a66dd3755d60686fc974e06", size = 117336, upload-time = "2025-04-02T10:06:23.14Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/b7/d59863ebe319150739d0c352c6dea2710a2f90254ed32304d52e8349edce/python_calamine-0.3.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5251746816069c38eafdd1e4eb7b83870e1fe0ff6191ce9a809b187ffba8ce93", size = 830854, upload-time = "2025-04-02T10:04:14.673Z" }, - { url = "https://files.pythonhosted.org/packages/d3/01/b48c6f2c2e530a1a031199c5c5bf35f7c2cf7f16f3989263e616e3bc86ce/python_calamine-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9775dbc93bc635d48f45433f8869a546cca28c2a86512581a05333f97a18337b", size = 809411, upload-time = "2025-04-02T10:04:16.067Z" }, - { url = "https://files.pythonhosted.org/packages/fe/6d/69c53ffb11b3ee1bf5bd945cc2514848adea492c879a50f38e2ed4424727/python_calamine-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ff4318b72ba78e8a04fb4c45342cfa23eab6f81ecdb85548cdab9f2db8ac9c7", size = 872905, upload-time = "2025-04-02T10:04:17.487Z" }, - { url = "https://files.pythonhosted.org/packages/be/ec/b02c4bc04c426d153af1f5ff07e797dd81ada6f47c170e0207d07c90b53a/python_calamine-0.3.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0cd8eb1ef8644da71788a33d3de602d1c08ff1c4136942d87e25f09580b512ef", size = 876464, upload-time = "2025-04-02T10:04:19.53Z" }, - { url = "https://files.pythonhosted.org/packages/46/ef/8403ee595207de5bd277279b56384b31390987df8a61c280b4176802481a/python_calamine-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9dcfd560d8f88f39d23b829f666ebae4bd8daeec7ed57adfb9313543f3c5fa35", size = 942289, upload-time = "2025-04-02T10:04:20.902Z" }, - { url = "https://files.pythonhosted.org/packages/89/97/b4e5b77c70b36613c10f2dbeece75b5d43727335a33bf5176792ec83c3fc/python_calamine-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5e79b9eae4b30c82d045f9952314137c7089c88274e1802947f9e3adb778a59", size = 978699, upload-time = "2025-04-02T10:04:22.263Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e9/03bbafd6b11cdf70c004f2e856978fc252ec5ea7e77529f14f969134c7a8/python_calamine-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce5e8cc518c8e3e5988c5c658f9dcd8229f5541ca63353175bb15b6ad8c456d0", size = 886008, upload-time = "2025-04-02T10:04:23.754Z" }, - { url = "https://files.pythonhosted.org/packages/7b/20/e18f534e49b403ba0b979a4dfead146001d867f5be846b91f81ed5377972/python_calamine-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a0e596b1346c28b2de15c9f86186cceefa4accb8882992aa0b7499c593446ed", size = 925104, upload-time = "2025-04-02T10:04:25.255Z" }, - { url = "https://files.pythonhosted.org/packages/54/4c/58933e69a0a7871487d10b958c1f83384bc430d53efbbfbf1dea141a0d85/python_calamine-0.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f521de16a9f3e951ec2e5e35d76752fe004088dbac4cdbf4dd62d0ad2bbf650f", size = 1050448, upload-time = "2025-04-02T10:04:26.649Z" }, - { url = "https://files.pythonhosted.org/packages/83/95/5c96d093eaaa2d15c63b43bcf8c87708eaab8428c72b6ebdcafc2604aa47/python_calamine-0.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:417d6825a36bba526ae17bed1b6ca576fbb54e23dc60c97eeb536c622e77c62f", size = 1056840, upload-time = "2025-04-02T10:04:28.18Z" }, - { url = "https://files.pythonhosted.org/packages/23/e0/b03cc3ad4f40fd3be0ebac0b71d273864ddf2bf0e611ec309328fdedded9/python_calamine-0.3.2-cp311-cp311-win32.whl", hash = "sha256:cd3ea1ca768139753633f9f0b16997648db5919894579f363d71f914f85f7ade", size = 663268, upload-time = "2025-04-02T10:04:29.659Z" }, - { url = "https://files.pythonhosted.org/packages/6b/bd/550da64770257fc70a185482f6353c0654a11f381227e146bb0170db040f/python_calamine-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:4560100412d8727c49048cca102eadeb004f91cfb9c99ae63cd7d4dc0a61333a", size = 692393, upload-time = "2025-04-02T10:04:31.534Z" }, - { url = "https://files.pythonhosted.org/packages/be/2e/0b4b7a146c3bb41116fe8e59a2f616340786db12aed51c7a9e75817cfa03/python_calamine-0.3.2-cp311-cp311-win_arm64.whl", hash = "sha256:a2526e6ba79087b1634f49064800339edb7316780dd7e1e86d10a0ca9de4e90f", size = 667312, upload-time = "2025-04-02T10:04:32.911Z" }, - { url = "https://files.pythonhosted.org/packages/f2/0f/c2e3e3bae774dae47cba6ffa640ff95525bd6a10a13d3cd998f33aeafc7f/python_calamine-0.3.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7c063b1f783352d6c6792305b2b0123784882e2436b638a9b9a1e97f6d74fa51", size = 825179, upload-time = "2025-04-02T10:04:34.377Z" }, - { url = "https://files.pythonhosted.org/packages/c7/81/a05285f06d71ea38ab99b09f3119f93f575487c9d24d7a1bab65657b258b/python_calamine-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85016728937e8f5d1810ff3c9603ffd2458d66e34d495202d7759fa8219871cd", size = 804036, upload-time = "2025-04-02T10:04:35.938Z" }, - { url = "https://files.pythonhosted.org/packages/24/b5/320f366ffd91ee5d5f0f77817d4fb684f62a5a68e438dcdb90e4f5f35137/python_calamine-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81f243323bf712bb0b2baf0b938a2e6d6c9fa3b9902a44c0654474d04f999fac", size = 871527, upload-time = "2025-04-02T10:04:38.272Z" }, - { url = "https://files.pythonhosted.org/packages/13/19/063afced19620b829697b90329c62ad73274cc38faaa91d9ee41047f5f8c/python_calamine-0.3.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0b719dd2b10237b0cfb2062e3eaf199f220918a5623197e8449f37c8de845a7c", size = 875411, upload-time = "2025-04-02T10:04:39.647Z" }, - { url = "https://files.pythonhosted.org/packages/d7/6a/c93c52414ec62cc51c4820aff434f03c4a1c69ced15cec3e4b93885e4012/python_calamine-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5158310b9140e8ee8665c9541a11030901e7275eb036988150c93f01c5133bf", size = 943525, upload-time = "2025-04-02T10:04:41.025Z" }, - { url = "https://files.pythonhosted.org/packages/0a/0a/5bdecee03d235e8d111b1e8ee3ea0c0ed4ae43a402f75cebbe719930cf04/python_calamine-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2c1b248e8bf10194c449cb57e6ccb3f2fe3dc86975a6d746908cf2d37b048cc", size = 976332, upload-time = "2025-04-02T10:04:42.454Z" }, - { url = "https://files.pythonhosted.org/packages/05/ad/43ff92366856ee34f958e9cf4f5b98e63b0dc219e06ccba4ad6f63463756/python_calamine-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a13ad8e5b6843a73933b8d1710bc4df39a9152cb57c11227ad51f47b5838a4", size = 885549, upload-time = "2025-04-02T10:04:43.869Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b9/76afb867e2bb4bfc296446b741cee01ae4ce6a094b43f4ed4eaed5189de4/python_calamine-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe950975a5758423c982ce1e2fdcb5c9c664d1a20b41ea21e619e5003bb4f96b", size = 926005, upload-time = "2025-04-02T10:04:45.884Z" }, - { url = "https://files.pythonhosted.org/packages/23/cf/5252b237b0e70c263f86741aea02e8e57aedb2bce9898468be1d9d55b9da/python_calamine-0.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8707622ba816d6c26e36f1506ecda66a6a6cf43e55a43a8ef4c3bf8a805d3cfb", size = 1049380, upload-time = "2025-04-02T10:04:49.202Z" }, - { url = "https://files.pythonhosted.org/packages/1a/4d/f151e8923e53457ca49ceeaa3a34cb23afee7d7b46e6546ab2a29adc9125/python_calamine-0.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e6eac46475c26e162a037f6711b663767f61f8fca3daffeb35aa3fc7ee6267cc", size = 1056720, upload-time = "2025-04-02T10:04:51.002Z" }, - { url = "https://files.pythonhosted.org/packages/f5/cb/1b5db3e4a8bbaaaa7706b270570d4a65133618fa0ca7efafe5ce680f6cee/python_calamine-0.3.2-cp312-cp312-win32.whl", hash = "sha256:0dee82aedef3db27368a388d6741d69334c1d4d7a8087ddd33f1912166e17e37", size = 663502, upload-time = "2025-04-02T10:04:52.402Z" }, - { url = "https://files.pythonhosted.org/packages/5a/53/920fa8e7b570647c08da0f1158d781db2e318918b06cb28fe0363c3398ac/python_calamine-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:ae09b779718809d31ca5d722464be2776b7d79278b1da56e159bbbe11880eecf", size = 692660, upload-time = "2025-04-02T10:04:53.721Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ea/5d0ecf5c345c4d78964a5f97e61848bc912965b276a54fb8ae698a9419a8/python_calamine-0.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:435546e401a5821fa70048b6c03a70db3b27d00037e2c4999c2126d8c40b51df", size = 666205, upload-time = "2025-04-02T10:04:56.377Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/cc/03/269f96535705b2f18c8977fa58e76763b4e4727a9b3ae277a9468c8ffe05/python_calamine-0.4.0.tar.gz", hash = "sha256:94afcbae3fec36d2d7475095a59d4dc6fae45829968c743cb799ebae269d7bbf", size = 127737, upload-time = "2025-07-04T06:05:28.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/a5/bcd82326d0ff1ab5889e7a5e13c868b483fc56398e143aae8e93149ba43b/python_calamine-0.4.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d1687f8c4d7852920c7b4e398072f183f88dd273baf5153391edc88b7454b8c0", size = 833019, upload-time = "2025-07-04T06:03:32.214Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1a/a681f1d2f28164552e91ef47bcde6708098aa64a5f5fe3952f22362d340a/python_calamine-0.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:258d04230bebbbafa370a15838049d912d6a0a2c4da128943d8160ca4b6db58e", size = 812268, upload-time = "2025-07-04T06:03:33.855Z" }, + { url = "https://files.pythonhosted.org/packages/3d/92/2fc911431733739d4e7a633cefa903fa49a6b7a61e8765bad29a4a7c47b1/python_calamine-0.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686e491634934f059553d55f77ac67ca4c235452d5b444f98fe79b3579f1ea5", size = 875733, upload-time = "2025-07-04T06:03:35.154Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/48bfae6802eb360028ca6c15e9edf42243aadd0006b6ac3e9edb41a57119/python_calamine-0.4.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4480af7babcc2f919c638a554b06b7b145d9ab3da47fd696d68c2fc6f67f9541", size = 878325, upload-time = "2025-07-04T06:03:36.638Z" }, + { url = "https://files.pythonhosted.org/packages/a4/dc/f8c956e15bac9d5d1e05cd1b907ae780e40522d2fd103c8c6e2f21dff4ed/python_calamine-0.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e405b87a8cd1e90a994e570705898634f105442029f25bab7da658ee9cbaa771", size = 1015038, upload-time = "2025-07-04T06:03:37.971Z" }, + { url = "https://files.pythonhosted.org/packages/54/3f/e69ab97c7734fb850fba2f506b775912fd59f04e17488582c8fbf52dbc72/python_calamine-0.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a831345ee42615f0dfcb0ed60a3b1601d2f946d4166edae64fd9a6f9bbd57fc1", size = 924969, upload-time = "2025-07-04T06:03:39.253Z" }, + { url = "https://files.pythonhosted.org/packages/79/03/b4c056b468908d87a3de94389166e0f4dba725a70bc39e03bc039ba96f6b/python_calamine-0.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9951b8e4cafb3e1623bb5dfc31a18d38ef43589275f9657e99dfcbe4c8c4b33e", size = 888020, upload-time = "2025-07-04T06:03:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/86/4f/b9092f7c970894054083656953184e44cb2dadff8852425e950d4ca419af/python_calamine-0.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a6619fe3b5c9633ed8b178684605f8076c9d8d85b29ade15f7a7713fcfdee2d0", size = 930337, upload-time = "2025-07-04T06:03:42.89Z" }, + { url = "https://files.pythonhosted.org/packages/64/da/137239027bf253aabe7063450950085ec9abd827d0cbc5170f585f38f464/python_calamine-0.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2cc45b8e76ee331f6ea88ca23677be0b7a05b502cd4423ba2c2bc8dad53af1be", size = 1054568, upload-time = "2025-07-04T06:03:44.153Z" }, + { url = "https://files.pythonhosted.org/packages/80/96/74c38bcf6b6825d5180c0e147b85be8c52dbfba11848b1e98ba358e32a64/python_calamine-0.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1b2cfb7ced1a7c80befa0cfddfe4aae65663eb4d63c4ae484b9b7a80ebe1b528", size = 1058317, upload-time = "2025-07-04T06:03:45.873Z" }, + { url = "https://files.pythonhosted.org/packages/33/95/9d7b8fe8b32d99a6c79534df3132cfe40e9df4a0f5204048bf5e66ddbd93/python_calamine-0.4.0-cp311-cp311-win32.whl", hash = "sha256:04f4e32ee16814fc1fafc49300be8eeb280d94878461634768b51497e1444bd6", size = 663934, upload-time = "2025-07-04T06:03:47.407Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e3/1c6cd9fd499083bea6ff1c30033ee8215b9f64e862babf5be170cacae190/python_calamine-0.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:a8543f69afac2213c0257bb56215b03dadd11763064a9d6b19786f27d1bef586", size = 692535, upload-time = "2025-07-04T06:03:48.699Z" }, + { url = "https://files.pythonhosted.org/packages/94/1c/3105d19fbab6b66874ce8831652caedd73b23b72e88ce18addf8ceca8c12/python_calamine-0.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:54622e35ec7c3b6f07d119da49aa821731c185e951918f152c2dbf3bec1e15d6", size = 671751, upload-time = "2025-07-04T06:03:49.979Z" }, + { url = "https://files.pythonhosted.org/packages/63/60/f951513aaaa470b3a38a87d65eca45e0a02bc329b47864f5a17db563f746/python_calamine-0.4.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:74bca5d44a73acf3dcfa5370820797fcfd225c8c71abcddea987c5b4f5077e98", size = 826603, upload-time = "2025-07-04T06:03:51.245Z" }, + { url = "https://files.pythonhosted.org/packages/76/3f/789955bbc77831c639890758f945eb2b25d6358065edf00da6751226cf31/python_calamine-0.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cf80178f5d1b0ee2ccfffb8549c50855f6249e930664adc5807f4d0d6c2b269c", size = 805826, upload-time = "2025-07-04T06:03:52.482Z" }, + { url = "https://files.pythonhosted.org/packages/00/4c/f87d17d996f647030a40bfd124fe45fe893c002bee35ae6aca9910a923ae/python_calamine-0.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65cfef345386ae86f7720f1be93495a40fd7e7feabb8caa1df5025d7fbc58a1f", size = 874989, upload-time = "2025-07-04T06:03:53.794Z" }, + { url = "https://files.pythonhosted.org/packages/47/d2/3269367303f6c0488cf1bfebded3f9fe968d118a988222e04c9b2636bf2e/python_calamine-0.4.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f23e6214dbf9b29065a5dcfd6a6c674dd0e251407298c9138611c907d53423ff", size = 877504, upload-time = "2025-07-04T06:03:55.095Z" }, + { url = "https://files.pythonhosted.org/packages/f9/6d/c7ac35f5c7125e8bd07eb36773f300fda20dd2da635eae78a8cebb0b6ab7/python_calamine-0.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d792d304ee232ab01598e1d3ab22e074a32c2511476b5fb4f16f4222d9c2a265", size = 1014171, upload-time = "2025-07-04T06:03:56.777Z" }, + { url = "https://files.pythonhosted.org/packages/f0/81/5ea8792a2e9ab5e2a05872db3a4d3ed3538ad5af1861282c789e2f13a8cf/python_calamine-0.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf813425918fd68f3e991ef7c4b5015be0a1a95fc4a8ab7e73c016ef1b881bb4", size = 926737, upload-time = "2025-07-04T06:03:58.024Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6e/989e56e6f073fc0981a74ba7a393881eb351bb143e5486aa629b5e5d6a8b/python_calamine-0.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbe2a0ccb4d003635888eea83a995ff56b0748c8c76fc71923544f5a4a7d4cd7", size = 887032, upload-time = "2025-07-04T06:03:59.298Z" }, + { url = "https://files.pythonhosted.org/packages/5d/92/2c9bd64277c6fe4be695d7d5a803b38d953ec8565037486be7506642c27c/python_calamine-0.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7b3bb5f0d910b9b03c240987560f843256626fd443279759df4e91b717826d2", size = 929700, upload-time = "2025-07-04T06:04:01.388Z" }, + { url = "https://files.pythonhosted.org/packages/64/fa/fc758ca37701d354a6bc7d63118699f1c73788a1f2e1b44d720824992764/python_calamine-0.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bd2c0fc2b5eabd08ceac8a2935bffa88dbc6116db971aa8c3f244bad3fd0f644", size = 1053971, upload-time = "2025-07-04T06:04:02.704Z" }, + { url = "https://files.pythonhosted.org/packages/65/52/40d7e08ae0ddba331cdc9f7fb3e92972f8f38d7afbd00228158ff6d1fceb/python_calamine-0.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:85b547cb1c5b692a0c2406678d666dbc1cec65a714046104683fe4f504a1721d", size = 1057057, upload-time = "2025-07-04T06:04:04.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/de/e8a071c0adfda73285d891898a24f6e99338328c404f497ff5b0e6bc3d45/python_calamine-0.4.0-cp312-cp312-win32.whl", hash = "sha256:4c2a1e3a0db4d6de4587999a21cc35845648c84fba81c03dd6f3072c690888e4", size = 665540, upload-time = "2025-07-04T06:04:05.679Z" }, + { url = "https://files.pythonhosted.org/packages/5e/f2/7fdfada13f80db12356853cf08697ff4e38800a1809c2bdd26ee60962e7a/python_calamine-0.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b193c89ffcc146019475cd121c552b23348411e19c04dedf5c766a20db64399a", size = 695366, upload-time = "2025-07-04T06:04:06.977Z" }, + { url = "https://files.pythonhosted.org/packages/20/66/d37412ad854480ce32f50d9f74f2a2f88b1b8a6fbc32f70aabf3211ae89e/python_calamine-0.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:43a0f15e0b60c75a71b21a012b911d5d6f5fa052afad2a8edbc728af43af0fcf", size = 670740, upload-time = "2025-07-04T06:04:08.656Z" }, ] [[package]] @@ -5066,27 +5088,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.11.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ed/da/9c6f995903b4d9474b39da91d2d626659af3ff1eeb43e9ae7c119349dba6/ruff-0.11.13.tar.gz", hash = "sha256:26fa247dc68d1d4e72c179e08889a25ac0c7ba4d78aecfc835d49cbfd60bf514", size = 4282054, upload-time = "2025-06-05T21:00:15.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/ce/a11d381192966e0b4290842cc8d4fac7dc9214ddf627c11c1afff87da29b/ruff-0.11.13-py3-none-linux_armv6l.whl", hash = "sha256:4bdfbf1240533f40042ec00c9e09a3aade6f8c10b6414cf11b519488d2635d46", size = 10292516, upload-time = "2025-06-05T20:59:32.944Z" }, - { url = "https://files.pythonhosted.org/packages/78/db/87c3b59b0d4e753e40b6a3b4a2642dfd1dcaefbff121ddc64d6c8b47ba00/ruff-0.11.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aef9c9ed1b5ca28bb15c7eac83b8670cf3b20b478195bd49c8d756ba0a36cf48", size = 11106083, upload-time = "2025-06-05T20:59:37.03Z" }, - { url = "https://files.pythonhosted.org/packages/77/79/d8cec175856ff810a19825d09ce700265f905c643c69f45d2b737e4a470a/ruff-0.11.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53b15a9dfdce029c842e9a5aebc3855e9ab7771395979ff85b7c1dedb53ddc2b", size = 10436024, upload-time = "2025-06-05T20:59:39.741Z" }, - { url = "https://files.pythonhosted.org/packages/8b/5b/f6d94f2980fa1ee854b41568368a2e1252681b9238ab2895e133d303538f/ruff-0.11.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab153241400789138d13f362c43f7edecc0edfffce2afa6a68434000ecd8f69a", size = 10646324, upload-time = "2025-06-05T20:59:42.185Z" }, - { url = "https://files.pythonhosted.org/packages/6c/9c/b4c2acf24ea4426016d511dfdc787f4ce1ceb835f3c5fbdbcb32b1c63bda/ruff-0.11.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c51f93029d54a910d3d24f7dd0bb909e31b6cd989a5e4ac513f4eb41629f0dc", size = 10174416, upload-time = "2025-06-05T20:59:44.319Z" }, - { url = "https://files.pythonhosted.org/packages/f3/10/e2e62f77c65ede8cd032c2ca39c41f48feabedb6e282bfd6073d81bb671d/ruff-0.11.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1808b3ed53e1a777c2ef733aca9051dc9bf7c99b26ece15cb59a0320fbdbd629", size = 11724197, upload-time = "2025-06-05T20:59:46.935Z" }, - { url = "https://files.pythonhosted.org/packages/bb/f0/466fe8469b85c561e081d798c45f8a1d21e0b4a5ef795a1d7f1a9a9ec182/ruff-0.11.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:d28ce58b5ecf0f43c1b71edffabe6ed7f245d5336b17805803312ec9bc665933", size = 12511615, upload-time = "2025-06-05T20:59:49.534Z" }, - { url = "https://files.pythonhosted.org/packages/17/0e/cefe778b46dbd0cbcb03a839946c8f80a06f7968eb298aa4d1a4293f3448/ruff-0.11.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55e4bc3a77842da33c16d55b32c6cac1ec5fb0fbec9c8c513bdce76c4f922165", size = 12117080, upload-time = "2025-06-05T20:59:51.654Z" }, - { url = "https://files.pythonhosted.org/packages/5d/2c/caaeda564cbe103bed145ea557cb86795b18651b0f6b3ff6a10e84e5a33f/ruff-0.11.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:633bf2c6f35678c56ec73189ba6fa19ff1c5e4807a78bf60ef487b9dd272cc71", size = 11326315, upload-time = "2025-06-05T20:59:54.469Z" }, - { url = "https://files.pythonhosted.org/packages/75/f0/782e7d681d660eda8c536962920c41309e6dd4ebcea9a2714ed5127d44bd/ruff-0.11.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ffbc82d70424b275b089166310448051afdc6e914fdab90e08df66c43bb5ca9", size = 11555640, upload-time = "2025-06-05T20:59:56.986Z" }, - { url = "https://files.pythonhosted.org/packages/5d/d4/3d580c616316c7f07fb3c99dbecfe01fbaea7b6fd9a82b801e72e5de742a/ruff-0.11.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a9ddd3ec62a9a89578c85842b836e4ac832d4a2e0bfaad3b02243f930ceafcc", size = 10507364, upload-time = "2025-06-05T20:59:59.154Z" }, - { url = "https://files.pythonhosted.org/packages/5a/dc/195e6f17d7b3ea6b12dc4f3e9de575db7983db187c378d44606e5d503319/ruff-0.11.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d237a496e0778d719efb05058c64d28b757c77824e04ffe8796c7436e26712b7", size = 10141462, upload-time = "2025-06-05T21:00:01.481Z" }, - { url = "https://files.pythonhosted.org/packages/f4/8e/39a094af6967faa57ecdeacb91bedfb232474ff8c3d20f16a5514e6b3534/ruff-0.11.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26816a218ca6ef02142343fd24c70f7cd8c5aa6c203bca284407adf675984432", size = 11121028, upload-time = "2025-06-05T21:00:04.06Z" }, - { url = "https://files.pythonhosted.org/packages/5a/c0/b0b508193b0e8a1654ec683ebab18d309861f8bd64e3a2f9648b80d392cb/ruff-0.11.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:51c3f95abd9331dc5b87c47ac7f376db5616041173826dfd556cfe3d4977f492", size = 11602992, upload-time = "2025-06-05T21:00:06.249Z" }, - { url = "https://files.pythonhosted.org/packages/7c/91/263e33ab93ab09ca06ce4f8f8547a858cc198072f873ebc9be7466790bae/ruff-0.11.13-py3-none-win32.whl", hash = "sha256:96c27935418e4e8e77a26bb05962817f28b8ef3843a6c6cc49d8783b5507f250", size = 10474944, upload-time = "2025-06-05T21:00:08.459Z" }, - { url = "https://files.pythonhosted.org/packages/46/f4/7c27734ac2073aae8efb0119cae6931b6fb48017adf048fdf85c19337afc/ruff-0.11.13-py3-none-win_amd64.whl", hash = "sha256:29c3189895a8a6a657b7af4e97d330c8a3afd2c9c8f46c81e2fc5a31866517e3", size = 11548669, upload-time = "2025-06-05T21:00:11.147Z" }, - { url = "https://files.pythonhosted.org/packages/ec/bf/b273dd11673fed8a6bd46032c0ea2a04b2ac9bfa9c628756a5856ba113b0/ruff-0.11.13-py3-none-win_arm64.whl", hash = "sha256:b4385285e9179d608ff1d2fb9922062663c658605819a6876d8beef0c30b7f3b", size = 10683928, upload-time = "2025-06-05T21:00:13.758Z" }, +version = "0.12.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/2a/43955b530c49684d3c38fcda18c43caf91e99204c2a065552528e0552d4f/ruff-0.12.3.tar.gz", hash = "sha256:f1b5a4b6668fd7b7ea3697d8d98857390b40c1320a63a178eee6be0899ea2d77", size = 4459341, upload-time = "2025-07-11T13:21:16.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/fd/b44c5115539de0d598d75232a1cc7201430b6891808df111b8b0506aae43/ruff-0.12.3-py3-none-linux_armv6l.whl", hash = "sha256:47552138f7206454eaf0c4fe827e546e9ddac62c2a3d2585ca54d29a890137a2", size = 10430499, upload-time = "2025-07-11T13:20:26.321Z" }, + { url = "https://files.pythonhosted.org/packages/43/c5/9eba4f337970d7f639a37077be067e4ec80a2ad359e4cc6c5b56805cbc66/ruff-0.12.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0a9153b000c6fe169bb307f5bd1b691221c4286c133407b8827c406a55282041", size = 11213413, upload-time = "2025-07-11T13:20:30.017Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2c/fac3016236cf1fe0bdc8e5de4f24c76ce53c6dd9b5f350d902549b7719b2/ruff-0.12.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fa6b24600cf3b750e48ddb6057e901dd5b9aa426e316addb2a1af185a7509882", size = 10586941, upload-time = "2025-07-11T13:20:33.046Z" }, + { url = "https://files.pythonhosted.org/packages/c5/0f/41fec224e9dfa49a139f0b402ad6f5d53696ba1800e0f77b279d55210ca9/ruff-0.12.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2506961bf6ead54887ba3562604d69cb430f59b42133d36976421bc8bd45901", size = 10783001, upload-time = "2025-07-11T13:20:35.534Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ca/dd64a9ce56d9ed6cad109606ac014860b1c217c883e93bf61536400ba107/ruff-0.12.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4faaff1f90cea9d3033cbbcdf1acf5d7fb11d8180758feb31337391691f3df0", size = 10269641, upload-time = "2025-07-11T13:20:38.459Z" }, + { url = "https://files.pythonhosted.org/packages/63/5c/2be545034c6bd5ce5bb740ced3e7014d7916f4c445974be11d2a406d5088/ruff-0.12.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40dced4a79d7c264389de1c59467d5d5cefd79e7e06d1dfa2c75497b5269a5a6", size = 11875059, upload-time = "2025-07-11T13:20:41.517Z" }, + { url = "https://files.pythonhosted.org/packages/8e/d4/a74ef1e801ceb5855e9527dae105eaff136afcb9cc4d2056d44feb0e4792/ruff-0.12.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:0262d50ba2767ed0fe212aa7e62112a1dcbfd46b858c5bf7bbd11f326998bafc", size = 12658890, upload-time = "2025-07-11T13:20:44.442Z" }, + { url = "https://files.pythonhosted.org/packages/13/c8/1057916416de02e6d7c9bcd550868a49b72df94e3cca0aeb77457dcd9644/ruff-0.12.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12371aec33e1a3758597c5c631bae9a5286f3c963bdfb4d17acdd2d395406687", size = 12232008, upload-time = "2025-07-11T13:20:47.374Z" }, + { url = "https://files.pythonhosted.org/packages/f5/59/4f7c130cc25220392051fadfe15f63ed70001487eca21d1796db46cbcc04/ruff-0.12.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:560f13b6baa49785665276c963edc363f8ad4b4fc910a883e2625bdb14a83a9e", size = 11499096, upload-time = "2025-07-11T13:20:50.348Z" }, + { url = "https://files.pythonhosted.org/packages/d4/01/a0ad24a5d2ed6be03a312e30d32d4e3904bfdbc1cdbe63c47be9d0e82c79/ruff-0.12.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023040a3499f6f974ae9091bcdd0385dd9e9eb4942f231c23c57708147b06311", size = 11688307, upload-time = "2025-07-11T13:20:52.945Z" }, + { url = "https://files.pythonhosted.org/packages/93/72/08f9e826085b1f57c9a0226e48acb27643ff19b61516a34c6cab9d6ff3fa/ruff-0.12.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:883d844967bffff5ab28bba1a4d246c1a1b2933f48cb9840f3fdc5111c603b07", size = 10661020, upload-time = "2025-07-11T13:20:55.799Z" }, + { url = "https://files.pythonhosted.org/packages/80/a0/68da1250d12893466c78e54b4a0ff381370a33d848804bb51279367fc688/ruff-0.12.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2120d3aa855ff385e0e562fdee14d564c9675edbe41625c87eeab744a7830d12", size = 10246300, upload-time = "2025-07-11T13:20:58.222Z" }, + { url = "https://files.pythonhosted.org/packages/6a/22/5f0093d556403e04b6fd0984fc0fb32fbb6f6ce116828fd54306a946f444/ruff-0.12.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6b16647cbb470eaf4750d27dddc6ebf7758b918887b56d39e9c22cce2049082b", size = 11263119, upload-time = "2025-07-11T13:21:01.503Z" }, + { url = "https://files.pythonhosted.org/packages/92/c9/f4c0b69bdaffb9968ba40dd5fa7df354ae0c73d01f988601d8fac0c639b1/ruff-0.12.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e1417051edb436230023575b149e8ff843a324557fe0a265863b7602df86722f", size = 11746990, upload-time = "2025-07-11T13:21:04.524Z" }, + { url = "https://files.pythonhosted.org/packages/fe/84/7cc7bd73924ee6be4724be0db5414a4a2ed82d06b30827342315a1be9e9c/ruff-0.12.3-py3-none-win32.whl", hash = "sha256:dfd45e6e926deb6409d0616078a666ebce93e55e07f0fb0228d4b2608b2c248d", size = 10589263, upload-time = "2025-07-11T13:21:07.148Z" }, + { url = "https://files.pythonhosted.org/packages/07/87/c070f5f027bd81f3efee7d14cb4d84067ecf67a3a8efb43aadfc72aa79a6/ruff-0.12.3-py3-none-win_amd64.whl", hash = "sha256:a946cf1e7ba3209bdef039eb97647f1c77f6f540e5845ec9c114d3af8df873e7", size = 11695072, upload-time = "2025-07-11T13:21:11.004Z" }, + { url = "https://files.pythonhosted.org/packages/e0/30/f3eaf6563c637b6e66238ed6535f6775480db973c836336e4122161986fc/ruff-0.12.3-py3-none-win_arm64.whl", hash = "sha256:5f9c7c9c8f84c2d7f27e93674d27136fbf489720251544c4da7fb3d742e011b1", size = 10805855, upload-time = "2025-07-11T13:21:13.547Z" }, ] [[package]] @@ -5297,6 +5319,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/fc/9ba22f01b5cdacc8f5ed0d22304718d2c758fce3fd49a5372b886a86f37c/sqlalchemy-2.0.41-py3-none-any.whl", hash = "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576", size = 1911224, upload-time = "2025-05-14T17:39:42.154Z" }, ] +[[package]] +name = "sseclient-py" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/ed/3df5ab8bb0c12f86c28d0cadb11ed1de44a92ed35ce7ff4fd5518a809325/sseclient-py-1.8.0.tar.gz", hash = "sha256:c547c5c1a7633230a38dc599a21a2dc638f9b5c297286b48b46b935c71fac3e8", size = 7791, upload-time = "2023-09-01T19:39:20.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/58/97655efdfeb5b4eeab85b1fc5d3fa1023661246c2ab2a26ea8e47402d4f2/sseclient_py-1.8.0-py2.py3-none-any.whl", hash = "sha256:4ecca6dc0b9f963f8384e9d7fd529bf93dd7d708144c4fb5da0e0a1a926fee83", size = 8828, upload-time = "2023-09-01T19:39:17.627Z" }, +] + [[package]] name = "starlette" version = "0.41.0" @@ -5599,11 +5630,11 @@ wheels = [ [[package]] name = "types-aiofiles" -version = "24.1.0.20250606" +version = "24.1.0.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/6e/fac4ffc896cb3faf2ac5d23747b65dd8bae1d9ee23305d1a3b12111c3989/types_aiofiles-24.1.0.20250606.tar.gz", hash = "sha256:48f9e26d2738a21e0b0f19381f713dcdb852a36727da8414b1ada145d40a18fe", size = 14364, upload-time = "2025-06-06T03:09:26.515Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/d6/5c44761bc11cb5c7505013a39f397a9016bfb3a5c932032b2db16c38b87b/types_aiofiles-24.1.0.20250708.tar.gz", hash = "sha256:c8207ed7385491ce5ba94da02658164ebd66b69a44e892288c9f20cbbf5284ff", size = 14322, upload-time = "2025-07-08T03:14:44.814Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/de/f2fa2ab8a5943898e93d8036941e05bfd1e1f377a675ee52c7c307dccb75/types_aiofiles-24.1.0.20250606-py3-none-any.whl", hash = "sha256:e568c53fb9017c80897a9aa15c74bf43b7ee90e412286ec1e0912b6e79301aee", size = 14276, upload-time = "2025-06-06T03:09:25.662Z" }, + { url = "https://files.pythonhosted.org/packages/44/e9/4e0cc79c630040aae0634ac9393341dc2aff1a5be454be9741cc6cc8989f/types_aiofiles-24.1.0.20250708-py3-none-any.whl", hash = "sha256:07f8f06465fd415d9293467d1c66cd074b2c3b62b679e26e353e560a8cf63720", size = 14320, upload-time = "2025-07-08T03:14:44.009Z" }, ] [[package]] @@ -5659,11 +5690,11 @@ wheels = [ [[package]] name = "types-defusedxml" -version = "0.7.0.20250516" +version = "0.7.0.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/9d/3ba8b80536402f1a125bc5a44d82ab686aafa55a85f56160e076b2ac30de/types_defusedxml-0.7.0.20250516.tar.gz", hash = "sha256:164c2945077fa450f24ed09633f8b3a80694687fefbbc1cba5f24e4ba570666b", size = 10298, upload-time = "2025-05-16T03:08:18.951Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/4b/79d046a7211e110afd885be04bb9423546df2a662ed28251512d60e51fb6/types_defusedxml-0.7.0.20250708.tar.gz", hash = "sha256:7b785780cc11c18a1af086308bf94bf53a0907943a1d145dbe00189bef323cb8", size = 10541, upload-time = "2025-07-08T03:14:33.325Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/7b/567b0978150edccf7fa3aa8f2566ea9c3ffc9481ce7d64428166934d6d7f/types_defusedxml-0.7.0.20250516-py3-none-any.whl", hash = "sha256:00e793e5c385c3e142d7c2acc3b4ccea2fe0828cee11e35501f0ba40386630a0", size = 12576, upload-time = "2025-05-16T03:08:17.892Z" }, + { url = "https://files.pythonhosted.org/packages/24/f8/870de7fbd5fee5643f05061db948df6bd574a05a42aee91e37ad47c999ef/types_defusedxml-0.7.0.20250708-py3-none-any.whl", hash = "sha256:cc426cbc31c61a0f1b1c2ad9b9ef9ef846645f28fd708cd7727a6353b5c52e54", size = 13478, upload-time = "2025-07-08T03:14:32.633Z" }, ] [[package]] @@ -5677,11 +5708,11 @@ wheels = [ [[package]] name = "types-docutils" -version = "0.21.0.20250604" +version = "0.21.0.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ef/d0/d28035370d669f14d4e23bd63d093207331f361afa24d2686d2c3fe6be8d/types_docutils-0.21.0.20250604.tar.gz", hash = "sha256:5a9cc7f5a4c5ef694aa0abc61111e0b1376a53dee90d65757f77f31acfcca8f2", size = 40953, upload-time = "2025-06-04T03:10:27.439Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/86/24394a71a04f416ca03df51863a3d3e2cd0542fdc40989188dca30ffb5bf/types_docutils-0.21.0.20250708.tar.gz", hash = "sha256:5625a82a9a2f26d8384545607c157e023a48ed60d940dfc738db125282864172", size = 42011, upload-time = "2025-07-08T03:14:24.214Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/91/887e9591c1ee50dfbf7c2fa2f3f51bc6db683013b6d2b0cd3983adf3d502/types_docutils-0.21.0.20250604-py3-none-any.whl", hash = "sha256:bfa8628176c06a80cdd1d6f3fb32e972e042db53538596488dfe0e9c5962b222", size = 65915, upload-time = "2025-06-04T03:10:26.067Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/8c1153fc1576a0dcffdd157c69a12863c3f9485054256f6791ea17d95aed/types_docutils-0.21.0.20250708-py3-none-any.whl", hash = "sha256:166630d1aec18b9ca02547873210e04bf7674ba8f8da9cd9e6a5e77dc99372c2", size = 67953, upload-time = "2025-07-08T03:14:23.057Z" }, ] [[package]] @@ -5733,11 +5764,11 @@ wheels = [ [[package]] name = "types-html5lib" -version = "1.1.11.20250516" +version = "1.1.11.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/ed/9f092ff479e2b5598941855f314a22953bb04b5fb38bcba3f880feb833ba/types_html5lib-1.1.11.20250516.tar.gz", hash = "sha256:65043a6718c97f7d52567cc0cdf41efbfc33b1f92c6c0c5e19f60a7ec69ae720", size = 16136, upload-time = "2025-05-16T03:07:12.231Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/3b/1f5ba4358cfc1421cced5cdb9d2b08b4b99e4f9a41da88ce079f6d1a7bf1/types_html5lib-1.1.11.20250708.tar.gz", hash = "sha256:24321720fdbac71cee50d5a4bec9b7448495b7217974cffe3fcf1ede4eef7afe", size = 16799, upload-time = "2025-07-08T03:13:53.14Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/3b/cb5b23c7b51bf48b8c9f175abb9dce2f1ecd2d2c25f92ea9f4e3720e9398/types_html5lib-1.1.11.20250516-py3-none-any.whl", hash = "sha256:5e407b14b1bd2b9b1107cbd1e2e19d4a0c46d60febd231c7ab7313d7405663c1", size = 21770, upload-time = "2025-05-16T03:07:11.102Z" }, + { url = "https://files.pythonhosted.org/packages/a8/50/5fc23cf647eee23acdd337c8150861d39980cf11f33dd87f78e87d2a4bad/types_html5lib-1.1.11.20250708-py3-none-any.whl", hash = "sha256:bb898066b155de7081cb182179e2ded31b9e0e234605e2cb46536894e68a6954", size = 22913, upload-time = "2025-07-08T03:13:52.098Z" }, ] [[package]] @@ -5856,11 +5887,11 @@ wheels = [ [[package]] name = "types-pymysql" -version = "1.1.0.20250516" +version = "1.1.0.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/db/11/cdaa90b82cb25c5e04e75f0b0616872aa5775b001096779375084f8dbbcf/types_pymysql-1.1.0.20250516.tar.gz", hash = "sha256:fea4a9776101cf893dfc868f42ce10d2e46dcc498c792cc7c9c0fe00cb744234", size = 19640, upload-time = "2025-05-16T03:06:54.568Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/a3/db349a06c64b8c041c165fc470b81d37404ec342014625c7a6b7f7a4f680/types_pymysql-1.1.0.20250708.tar.gz", hash = "sha256:2cbd7cfcf9313eda784910578c4f1d06f8cc03a15cd30ce588aa92dd6255011d", size = 21715, upload-time = "2025-07-08T03:13:56.463Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/64/129656e04ddda35d69faae914ce67cf60d83407ddd7afdef1e7c50bbb74a/types_pymysql-1.1.0.20250516-py3-none-any.whl", hash = "sha256:41c87a832e3ff503d5120cc6cebd64f6dcb3c407d9580a98b2cb3e3bcd109aa6", size = 20328, upload-time = "2025-05-16T03:06:53.681Z" }, + { url = "https://files.pythonhosted.org/packages/88/e5/7f72c520f527175b6455e955426fd4f971128b4fa2f8ab2f505f254a1ddc/types_pymysql-1.1.0.20250708-py3-none-any.whl", hash = "sha256:9252966d2795945b2a7a53d5cdc49fe8e4e2f3dde4c104ed7fc782a83114e365", size = 22860, upload-time = "2025-07-08T03:13:55.367Z" }, ] [[package]] @@ -5878,20 +5909,20 @@ wheels = [ [[package]] name = "types-python-dateutil" -version = "2.9.0.20250516" +version = "2.9.0.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ef/88/d65ed807393285204ab6e2801e5d11fbbea811adcaa979a2ed3b67a5ef41/types_python_dateutil-2.9.0.20250516.tar.gz", hash = "sha256:13e80d6c9c47df23ad773d54b2826bd52dbbb41be87c3f339381c1700ad21ee5", size = 13943, upload-time = "2025-05-16T03:06:58.385Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/95/6bdde7607da2e1e99ec1c1672a759d42f26644bbacf939916e086db34870/types_python_dateutil-2.9.0.20250708.tar.gz", hash = "sha256:ccdbd75dab2d6c9696c350579f34cffe2c281e4c5f27a585b2a2438dd1d5c8ab", size = 15834, upload-time = "2025-07-08T03:14:03.382Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/3f/b0e8db149896005adc938a1e7f371d6d7e9eca4053a29b108978ed15e0c2/types_python_dateutil-2.9.0.20250516-py3-none-any.whl", hash = "sha256:2b2b3f57f9c6a61fba26a9c0ffb9ea5681c9b83e69cd897c6b5f668d9c0cab93", size = 14356, upload-time = "2025-05-16T03:06:57.249Z" }, + { url = "https://files.pythonhosted.org/packages/72/52/43e70a8e57fefb172c22a21000b03ebcc15e47e97f5cb8495b9c2832efb4/types_python_dateutil-2.9.0.20250708-py3-none-any.whl", hash = "sha256:4d6d0cc1cc4d24a2dc3816024e502564094497b713f7befda4d5bc7a8e3fd21f", size = 17724, upload-time = "2025-07-08T03:14:02.593Z" }, ] [[package]] name = "types-python-http-client" -version = "3.3.7.20240910" +version = "3.3.7.20250708" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/d7/bb2754c2d1b20c1890593ec89799c99e8875b04f474197c41354f41e9d31/types-python-http-client-3.3.7.20240910.tar.gz", hash = "sha256:8a6ebd30ad4b90a329ace69c240291a6176388624693bc971a5ecaa7e9b05074", size = 2804, upload-time = "2024-09-10T02:38:31.608Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/a0/0ad93698a3ebc6846ca23aca20ff6f6f8ebe7b4f0c1de7f19e87c03dbe8f/types_python_http_client-3.3.7.20250708.tar.gz", hash = "sha256:5f85b32dc64671a4e5e016142169aa187c5abed0b196680944e4efd3d5ce3322", size = 7707, upload-time = "2025-07-08T03:14:36.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/95/8f492d37d99630e096acbb4071788483282a34a73ae89dd1a5727f4189cc/types_python_http_client-3.3.7.20240910-py3-none-any.whl", hash = "sha256:58941bd986fb8bb0f4f782ef376be145ece8023f391364fbcd22bd26b13a140e", size = 3917, upload-time = "2024-09-10T02:38:30.261Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/b88274658cf489e35175be8571c970e9a1219713bafd8fc9e166d7351ecb/types_python_http_client-3.3.7.20250708-py3-none-any.whl", hash = "sha256:e2fc253859decab36713d82fc7f205868c3ddeaee79dbb55956ad9ca77abe12b", size = 8890, upload-time = "2025-07-08T03:14:35.506Z" }, ] [[package]] @@ -6040,11 +6071,11 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.14.0" +version = "4.14.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" }, + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, ] [[package]] @@ -6172,7 +6203,7 @@ pptx = [ [[package]] name = "unstructured-client" -version = "0.37.4" +version = "0.38.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -6183,9 +6214,9 @@ dependencies = [ { name = "pypdf" }, { name = "requests-toolbelt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/6f/8dd20dab879f25074d6abfbb98f77bb8efeea0ae1bdf9a414b3e73c152b6/unstructured_client-0.37.4.tar.gz", hash = "sha256:5a4029563c2f79de098374fd8a99090719df325b4bdcfa3a87820908f2c83e6c", size = 90481, upload-time = "2025-07-01T16:40:09.877Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/60/412092671bfc4952640739f2c0c9b2f4c8af26a3c921738fd12621b4ddd8/unstructured_client-0.38.1.tar.gz", hash = "sha256:43ab0670dd8ff53d71e74f9b6dfe490a84a5303dab80a4873e118a840c6d46ca", size = 91781, upload-time = "2025-07-03T15:46:35.054Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/09/4399b0c32564b1a19fef943b5acea5a16fa0c6aa7a320065ce726b8245c1/unstructured_client-0.37.4-py3-none-any.whl", hash = "sha256:31975c0ea4408e369e6aad11c9e746d1f3f14013ac5c89f9f8dbada3a21dcec0", size = 211242, upload-time = "2025-07-01T16:40:08.642Z" }, + { url = "https://files.pythonhosted.org/packages/26/e0/8c249f00ba85fb4aba5c541463312befbfbf491105ff5c06e508089467be/unstructured_client-0.38.1-py3-none-any.whl", hash = "sha256:71e5467870d0a0119c788c29ec8baf5c0f7123f424affc9d6682eeeb7b8d45fa", size = 212626, upload-time = "2025-07-03T15:46:33.929Z" }, ] [[package]] @@ -6220,11 +6251,11 @@ wheels = [ [[package]] name = "uuid6" -version = "2025.0.0" +version = "2025.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/49/06a089c184580f510e20226d9a081e4323d13db2fbc92d566697b5395c1e/uuid6-2025.0.0.tar.gz", hash = "sha256:bb78aa300e29db89b00410371d0c1f1824e59e29995a9daa3dedc8033d1d84ec", size = 13941, upload-time = "2025-06-11T20:02:05.324Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/b7/4c0f736ca824b3a25b15e8213d1bcfc15f8ac2ae48d1b445b310892dc4da/uuid6-2025.0.1.tar.gz", hash = "sha256:cd0af94fa428675a44e32c5319ec5a3485225ba2179eefcf4c3f205ae30a81bd", size = 13932, upload-time = "2025-07-04T18:30:35.186Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/50/4da47101af45b6cfa291559577993b52ee4399b3cd54ba307574a11e4f3a/uuid6-2025.0.0-py3-none-any.whl", hash = "sha256:2c73405ff5333c7181443958c6865e0d1b9b816bb160549e8d80ba186263cb3a", size = 7001, upload-time = "2025-06-11T20:02:04.521Z" }, + { url = "https://files.pythonhosted.org/packages/3d/b2/93faaab7962e2aa8d6e174afb6f76be2ca0ce89fde14d3af835acebcaa59/uuid6-2025.0.1-py3-none-any.whl", hash = "sha256:80530ce4d02a93cdf82e7122ca0da3ebbbc269790ec1cb902481fa3e9cc9ff99", size = 6979, upload-time = "2025-07-04T18:30:34.001Z" }, ] [[package]] diff --git a/docker/.env.example b/docker/.env.example index a403f25cb2..dabd66f285 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -47,6 +47,11 @@ APP_WEB_URL= # ensuring port 5001 is externally accessible (see docker-compose.yaml). FILES_URL= +# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network. +# Set this to the internal Docker service URL for proper plugin file access. +# Example: INTERNAL_FILES_URL=http://api:5001 +INTERNAL_FILES_URL= + # ------------------------------ # Server Configuration # ------------------------------ @@ -794,6 +799,19 @@ WORKFLOW_FILE_UPLOAD_LIMIT=10 # hybrid: Save new data to object storage, read from both object storage and RDBMS WORKFLOW_NODE_EXECUTION_STORAGE=rdbms +# Repository configuration +# Core workflow execution repository implementation +CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository + +# Core workflow node execution repository implementation +CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository + +# API workflow node execution repository implementation +API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository + +# API workflow run repository implementation +API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository + # HTTP request node in workflow configuration HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index fd7c78c7e7..7c1544acb9 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.6.0 restart: always environment: # Use the shared environment variables. @@ -31,7 +31,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.6.0 restart: always environment: # Use the shared environment variables. @@ -57,7 +57,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.5.1 + image: langgenius/dify-web:1.6.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 0a95251ff0..61362ed9fd 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -11,6 +11,7 @@ x-shared-env: &shared-api-worker-env APP_API_URL: ${APP_API_URL:-} APP_WEB_URL: ${APP_WEB_URL:-} FILES_URL: ${FILES_URL:-} + INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} LOG_LEVEL: ${LOG_LEVEL:-INFO} LOG_FILE: ${LOG_FILE:-/app/logs/server.log} LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20} @@ -353,6 +354,10 @@ x-shared-env: &shared-api-worker-env WORKFLOW_PARALLEL_DEPTH_LIMIT: ${WORKFLOW_PARALLEL_DEPTH_LIMIT:-3} WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10} WORKFLOW_NODE_EXECUTION_STORAGE: ${WORKFLOW_NODE_EXECUTION_STORAGE:-rdbms} + CORE_WORKFLOW_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository} + CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository} + API_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${API_WORKFLOW_NODE_EXECUTION_REPOSITORY:-repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository} + API_WORKFLOW_RUN_REPOSITORY: ${API_WORKFLOW_RUN_REPOSITORY:-repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository} HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} @@ -518,7 +523,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.6.0 restart: always environment: # Use the shared environment variables. @@ -547,7 +552,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.6.0 restart: always environment: # Use the shared environment variables. @@ -573,7 +578,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.5.1 + image: langgenius/dify-web:1.6.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/docker/nginx/conf.d/default.conf.template b/docker/nginx/conf.d/default.conf.template index a458412d1e..48d7da8cf5 100644 --- a/docker/nginx/conf.d/default.conf.template +++ b/docker/nginx/conf.d/default.conf.template @@ -39,7 +39,10 @@ server { proxy_pass http://web:3000; include proxy.conf; } - + location /mcp { + proxy_pass http://api:5001; + include proxy.conf; + } # placeholder for acme challenge location ${ACME_CHALLENGE_LOCATION} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx index 084adceef2..3d572b926a 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx @@ -5,6 +5,7 @@ import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import AppCard from '@/app/components/app/overview/appCard' import Loading from '@/app/components/base/loading' +import MCPServiceCard from '@/app/components/tools/mcp/mcp-service-card' import { ToastContext } from '@/app/components/base/toast' import { fetchAppDetail, @@ -31,6 +32,8 @@ const CardView: FC = ({ appId, isInPanel, className }) => { const appDetail = useAppStore(state => state.appDetail) const setAppDetail = useAppStore(state => state.setAppDetail) + const showMCPCard = isInPanel + const updateAppDetail = async () => { try { const res = await fetchAppDetail({ url: '/apps', id: appId }) @@ -117,6 +120,11 @@ const CardView: FC = ({ appId, isInPanel, className }) => { isInPanel={isInPanel} onChangeStatus={onChangeApiStatus} /> + {showMCPCard && ( + + )} ) } diff --git a/web/app/(shareLayout)/webapp-reset-password/set-password/page.tsx b/web/app/(shareLayout)/webapp-reset-password/set-password/page.tsx index 9f9a8ad4e3..5e3f6fff1d 100644 --- a/web/app/(shareLayout)/webapp-reset-password/set-password/page.tsx +++ b/web/app/(shareLayout)/webapp-reset-password/set-password/page.tsx @@ -9,8 +9,7 @@ import Button from '@/app/components/base/button' import { changeWebAppPasswordWithToken } from '@/service/common' import Toast from '@/app/components/base/toast' import Input from '@/app/components/base/input' - -const validPassword = /^(?=.*[a-zA-Z])(?=.*\d).{8,}$/ +import { validPassword } from '@/config' const ChangePasswordForm = () => { const { t } = useTranslation() diff --git a/web/app/account/account-page/index.tsx b/web/app/account/account-page/index.tsx index 19c1e44236..ffaecb79ef 100644 --- a/web/app/account/account-page/index.tsx +++ b/web/app/account/account-page/index.tsx @@ -21,6 +21,7 @@ import { IS_CE_EDITION } from '@/config' import Input from '@/app/components/base/input' import PremiumBadge from '@/app/components/base/premium-badge' import { useGlobalPublicStore } from '@/context/global-public-context' +import { validPassword } from '@/config' const titleClassName = ` system-sm-semibold text-text-secondary @@ -29,8 +30,6 @@ const descriptionClassName = ` mt-1 body-xs-regular text-text-tertiary ` -const validPassword = /^(?=.*[a-zA-Z])(?=.*\d).{8,}$/ - export default function AccountPage() { const { t } = useTranslation() const { systemFeatures } = useGlobalPublicStore() diff --git a/web/app/components/app-sidebar/app-info.tsx b/web/app/components/app-sidebar/app-info.tsx index c28cc20df5..3817ebf5a4 100644 --- a/web/app/components/app-sidebar/app-info.tsx +++ b/web/app/components/app-sidebar/app-info.tsx @@ -308,13 +308,11 @@ const AppInfo = ({ expand, onlyShowDetail = false, openState = false, onDetailEx operations={operations} /> -
- -
+
- {isExtraInLine ? ( + {!hideType && isExtraInLine && (
{type}
- ) : ( + )} + {!hideType && !isExtraInLine && (
{isExternal ? t('dataset.externalTag') : type}
)} } diff --git a/web/app/components/app/configuration/config-var/config-modal/index.tsx b/web/app/components/app/configuration/config-var/config-modal/index.tsx index 29cbc55b90..8fcc0f4c08 100644 --- a/web/app/components/app/configuration/config-var/config-modal/index.tsx +++ b/web/app/components/app/configuration/config-var/config-modal/index.tsx @@ -1,5 +1,5 @@ 'use client' -import type { FC } from 'react' +import type { ChangeEvent, FC } from 'react' import React, { useCallback, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' @@ -11,7 +11,7 @@ import SelectTypeItem from '../select-type-item' import Field from './field' import Input from '@/app/components/base/input' import Toast from '@/app/components/base/toast' -import { checkKeys, getNewVarInWorkflow } from '@/utils/var' +import { checkKeys, getNewVarInWorkflow, replaceSpaceWithUnderscreInVarNameInput } from '@/utils/var' import ConfigContext from '@/context/debug-configuration' import type { InputVar, MoreInfo, UploadFileSetting } from '@/app/components/workflow/types' import Modal from '@/app/components/base/modal' @@ -109,6 +109,20 @@ const ConfigModal: FC = ({ }) }, [checkVariableName, tempPayload.label]) + const handleVarNameChange = useCallback((e: ChangeEvent) => { + replaceSpaceWithUnderscreInVarNameInput(e.target) + const value = e.target.value + const { isValid, errorKey, errorMessageKey } = checkKeys([value], true) + if (!isValid) { + Toast.notify({ + type: 'error', + message: t(`appDebug.varKeyError.${errorMessageKey}`, { key: errorKey }), + }) + return + } + handlePayloadChange('variable')(e.target.value) + }, [handlePayloadChange, t]) + const handleConfirm = () => { const moreInfo = tempPayload.variable === payload?.variable ? undefined @@ -200,7 +214,7 @@ const ConfigModal: FC = ({ handlePayloadChange('variable')(e.target.value)} + onChange={handleVarNameChange} onBlur={handleVarKeyBlur} placeholder={t('appDebug.variableConfig.inputPlaceholder')!} /> diff --git a/web/app/components/app/configuration/config/agent/agent-tools/index.tsx b/web/app/components/app/configuration/config/agent/agent-tools/index.tsx index a3149447d4..66fe85a170 100644 --- a/web/app/components/app/configuration/config/agent/agent-tools/index.tsx +++ b/web/app/components/app/configuration/config/agent/agent-tools/index.tsx @@ -30,15 +30,31 @@ import ConfigCredential from '@/app/components/tools/setting/build-in/config-cre import { updateBuiltInToolCredential } from '@/service/tools' import cn from '@/utils/classnames' import ToolPicker from '@/app/components/workflow/block-selector/tool-picker' -import type { ToolDefaultValue } from '@/app/components/workflow/block-selector/types' +import type { ToolDefaultValue, ToolValue } from '@/app/components/workflow/block-selector/types' import { canFindTool } from '@/utils' +import { useAllBuiltInTools, useAllCustomTools, useAllMCPTools, useAllWorkflowTools } from '@/service/use-tools' +import type { ToolWithProvider } from '@/app/components/workflow/types' import { useMittContextSelector } from '@/context/mitt-context' type AgentToolWithMoreInfo = AgentTool & { icon: any; collection?: Collection } | null const AgentTools: FC = () => { const { t } = useTranslation() const [isShowChooseTool, setIsShowChooseTool] = useState(false) - const { modelConfig, setModelConfig, collectionList } = useContext(ConfigContext) + const { modelConfig, setModelConfig } = useContext(ConfigContext) + const { data: buildInTools } = useAllBuiltInTools() + const { data: customTools } = useAllCustomTools() + const { data: workflowTools } = useAllWorkflowTools() + const { data: mcpTools } = useAllMCPTools() + const collectionList = useMemo(() => { + const allTools = [ + ...(buildInTools || []), + ...(customTools || []), + ...(workflowTools || []), + ...(mcpTools || []), + ] + return allTools + }, [buildInTools, customTools, workflowTools, mcpTools]) + const formattingChangedDispatcher = useFormattingChangedDispatcher() const [currentTool, setCurrentTool] = useState(null) const currentCollection = useMemo(() => { @@ -96,23 +112,38 @@ const AgentTools: FC = () => { } const [isDeleting, setIsDeleting] = useState(-1) - + const getToolValue = (tool: ToolDefaultValue) => { + return { + provider_id: tool.provider_id, + provider_type: tool.provider_type as CollectionType, + provider_name: tool.provider_name, + tool_name: tool.tool_name, + tool_label: tool.tool_label, + tool_parameters: tool.params, + notAuthor: !tool.is_team_authorization, + enabled: true, + } + } const handleSelectTool = (tool: ToolDefaultValue) => { const newModelConfig = produce(modelConfig, (draft) => { - draft.agentConfig.tools.push({ - provider_id: tool.provider_id, - provider_type: tool.provider_type as CollectionType, - provider_name: tool.provider_name, - tool_name: tool.tool_name, - tool_label: tool.tool_label, - tool_parameters: tool.params, - notAuthor: !tool.is_team_authorization, - enabled: true, - }) + draft.agentConfig.tools.push(getToolValue(tool)) }) setModelConfig(newModelConfig) } + const handleSelectMultipleTool = (tool: ToolDefaultValue[]) => { + const newModelConfig = produce(modelConfig, (draft) => { + draft.agentConfig.tools.push(...tool.map(getToolValue)) + }) + setModelConfig(newModelConfig) + } + const getProviderShowName = (item: AgentTool) => { + const type = item.provider_type + if(type === CollectionType.builtIn) + return item.provider_name.split('/').pop() + return item.provider_name + } + return ( <> { disabled={false} supportAddCustomTool onSelect={handleSelectTool} - selectedTools={tools as any} + onSelectMultiple={handleSelectMultipleTool} + selectedTools={tools as unknown as ToolValue[]} + canChooseMCPTool /> )} @@ -161,7 +194,7 @@ const AgentTools: FC = () => {
{item.isDeleted && } {!item.isDeleted && ( -
+
{typeof item.icon === 'string' &&
} {typeof item.icon !== 'string' && }
@@ -172,7 +205,7 @@ const AgentTools: FC = () => { (item.isDeleted || item.notAuthor || !item.enabled) ? 'opacity-50' : '', )} > - {item.provider_type === CollectionType.builtIn ? item.provider_name.split('/').pop() : item.tool_label} + {getProviderShowName(item)} {item.tool_label} {!item.isDeleted && ( { setIsShowSettingTool(false)} diff --git a/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx b/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx index 952ad66fc4..1ad814c6e9 100644 --- a/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx +++ b/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx @@ -24,10 +24,11 @@ import { fetchBuiltInToolList, fetchCustomToolList, fetchModelToolList, fetchWor import I18n from '@/context/i18n' import { getLanguage } from '@/i18n/language' import cn from '@/utils/classnames' +import type { ToolWithProvider } from '@/app/components/workflow/types' type Props = { showBackButton?: boolean - collection: Collection + collection: Collection | ToolWithProvider isBuiltIn?: boolean isModel?: boolean toolName: string @@ -51,9 +52,10 @@ const SettingBuiltInTool: FC = ({ const { locale } = useContext(I18n) const language = getLanguage(locale) const { t } = useTranslation() - - const [isLoading, setIsLoading] = useState(true) - const [tools, setTools] = useState([]) + const passedTools = (collection as ToolWithProvider).tools + const hasPassedTools = passedTools?.length > 0 + const [isLoading, setIsLoading] = useState(!hasPassedTools) + const [tools, setTools] = useState(hasPassedTools ? passedTools : []) const currTool = tools.find(tool => tool.name === toolName) const formSchemas = currTool ? toolParametersToFormSchemas(currTool.parameters) : [] const infoSchemas = formSchemas.filter(item => item.form === 'llm') @@ -63,7 +65,7 @@ const SettingBuiltInTool: FC = ({ const [currType, setCurrType] = useState('info') const isInfoActive = currType === 'info' useEffect(() => { - if (!collection) + if (!collection || hasPassedTools) return (async () => { diff --git a/web/app/components/app/configuration/config/config-audio.tsx b/web/app/components/app/configuration/config/config-audio.tsx new file mode 100644 index 0000000000..5600f8cbb6 --- /dev/null +++ b/web/app/components/app/configuration/config/config-audio.tsx @@ -0,0 +1,78 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import produce from 'immer' +import { useContext } from 'use-context-selector' + +import { Microphone01 } from '@/app/components/base/icons/src/vender/features' +import Tooltip from '@/app/components/base/tooltip' +import ConfigContext from '@/context/debug-configuration' +import { SupportUploadFileTypes } from '@/app/components/workflow/types' +import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks' +import Switch from '@/app/components/base/switch' + +const ConfigAudio: FC = () => { + const { t } = useTranslation() + const file = useFeatures(s => s.features.file) + const featuresStore = useFeaturesStore() + const { isShowAudioConfig } = useContext(ConfigContext) + + const isAudioEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.audio) ?? false + + const handleChange = useCallback((value: boolean) => { + const { + features, + setFeatures, + } = featuresStore!.getState() + + const newFeatures = produce(features, (draft) => { + if (value) { + draft.file!.allowed_file_types = Array.from(new Set([ + ...(draft.file?.allowed_file_types || []), + SupportUploadFileTypes.audio, + ])) + } + else { + draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter( + type => type !== SupportUploadFileTypes.audio, + ) + } + if (draft.file) + draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0 + }) + setFeatures(newFeatures) + }, [featuresStore]) + + if (!isShowAudioConfig) + return null + + return ( +
+
+
+ +
+
+
+
{t('appDebug.feature.audioUpload.title')}
+ + {t('appDebug.feature.audioUpload.description')} +
+ } + /> +
+
+
+ +
+
+ ) +} +export default React.memo(ConfigAudio) diff --git a/web/app/components/app/configuration/config/index.tsx b/web/app/components/app/configuration/config/index.tsx index dc2095502e..d0375c6de9 100644 --- a/web/app/components/app/configuration/config/index.tsx +++ b/web/app/components/app/configuration/config/index.tsx @@ -8,6 +8,7 @@ import DatasetConfig from '../dataset-config' import HistoryPanel from '../config-prompt/conversation-history/history-panel' import ConfigVision from '../config-vision' import ConfigDocument from './config-document' +import ConfigAudio from './config-audio' import AgentTools from './agent/agent-tools' import ConfigContext from '@/context/debug-configuration' import ConfigPrompt from '@/app/components/app/configuration/config-prompt' @@ -85,6 +86,8 @@ const Config: FC = () => { + + {/* Chat History */} {isAdvancedMode && isChatApp && modelModeType === ModelModeType.completion && ( { const isShowVisionConfig = !!currModel?.features?.includes(ModelFeatureEnum.vision) const isShowDocumentConfig = !!currModel?.features?.includes(ModelFeatureEnum.document) + const isShowAudioConfig = !!currModel?.features?.includes(ModelFeatureEnum.audio) const isAllowVideoUpload = !!currModel?.features?.includes(ModelFeatureEnum.video) // *** web app features *** const featuresData: FeaturesData = useMemo(() => { @@ -920,6 +921,7 @@ const Configuration: FC = () => { setVisionConfig: handleSetVisionConfig, isAllowVideoUpload, isShowDocumentConfig, + isShowAudioConfig, rerankSettingModalOpen, setRerankSettingModalOpen, }} diff --git a/web/app/components/app/log/list.tsx b/web/app/components/app/log/list.tsx index 47f8c09e39..b04148d484 100644 --- a/web/app/components/app/log/list.tsx +++ b/web/app/components/app/log/list.tsx @@ -470,8 +470,6 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) { className="py-4" id="scrollableDiv" style={{ - height: 1000, // Specify a value - overflow: 'auto', display: 'flex', flexDirection: 'column-reverse', }}> diff --git a/web/app/components/app/overview/appCard.tsx b/web/app/components/app/overview/appCard.tsx index 9f3b3ac4a6..f11e111cb0 100644 --- a/web/app/components/app/overview/appCard.tsx +++ b/web/app/components/app/overview/appCard.tsx @@ -181,6 +181,7 @@ function AppCard({ icon={appInfo.icon} icon_background={appInfo.icon_background} name={basicName} + hideType type={ isApp ? t('appOverview.overview.appInfo.explanation') diff --git a/web/app/components/base/app-icon/index.tsx b/web/app/components/base/app-icon/index.tsx index ac17af1988..003d929c8c 100644 --- a/web/app/components/base/app-icon/index.tsx +++ b/web/app/components/base/app-icon/index.tsx @@ -18,6 +18,7 @@ export type AppIconProps = { imageUrl?: string | null className?: string innerIcon?: React.ReactNode + coverElement?: React.ReactNode onClick?: () => void } const appIconVariants = cva( @@ -51,6 +52,7 @@ const AppIcon: FC = ({ imageUrl, className, innerIcon, + coverElement, onClick, }) => { const isValidImageIcon = iconType === 'image' && imageUrl @@ -65,6 +67,7 @@ const AppIcon: FC = ({ ? app icon : (innerIcon || ((icon && icon !== '') ? : )) } + {coverElement} } diff --git a/web/app/components/base/chat/chat/hooks.ts b/web/app/components/base/chat/chat/hooks.ts index 10fb455d33..17373cec9d 100644 --- a/web/app/components/base/chat/chat/hooks.ts +++ b/web/app/components/base/chat/chat/hooks.ts @@ -83,12 +83,11 @@ export const useChat = ( const ret = [...threadMessages] if (config?.opening_statement) { const index = threadMessages.findIndex(item => item.isOpeningStatement) - if (index > -1) { ret[index] = { ...ret[index], content: getIntroduction(config.opening_statement), - suggestedQuestions: config.suggested_questions, + suggestedQuestions: config.suggested_questions?.map(item => getIntroduction(item)), } } else { @@ -97,7 +96,7 @@ export const useChat = ( content: getIntroduction(config.opening_statement), isAnswer: true, isOpeningStatement: true, - suggestedQuestions: config.suggested_questions, + suggestedQuestions: config.suggested_questions?.map(item => getIntroduction(item)), }) } } diff --git a/web/app/components/base/chat/chat/question.tsx b/web/app/components/base/chat/chat/question.tsx index 30077125f9..d221587940 100644 --- a/web/app/components/base/chat/chat/question.tsx +++ b/web/app/components/base/chat/chat/question.tsx @@ -98,7 +98,7 @@ const Question: FC = ({ return (
-
+
= ({
-
+
{isSearching && <>

Search

@@ -170,7 +170,7 @@ const EmojiPickerInner: FC = ({ 'flex h-8 w-8 items-center justify-center rounded-lg p-1', ) } style={{ background: color }}> - {selectedEmoji !== '' && } + {selectedEmoji !== '' && }
})} diff --git a/web/app/components/base/features/new-feature-panel/conversation-opener/modal.tsx b/web/app/components/base/features/new-feature-panel/conversation-opener/modal.tsx index 117c8a5558..51e33c43d2 100644 --- a/web/app/components/base/features/new-feature-panel/conversation-opener/modal.tsx +++ b/web/app/components/base/features/new-feature-panel/conversation-opener/modal.tsx @@ -130,6 +130,7 @@ const OpeningSettingModal = ({ { const value = e.target.value setTempSuggestedQuestions(tempSuggestedQuestions.map((item, i) => { diff --git a/web/app/components/base/icons/assets/vender/line/others/search-menu.svg b/web/app/components/base/icons/assets/vender/line/others/search-menu.svg new file mode 100644 index 0000000000..f61f69f4ba --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/others/search-menu.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/web/app/components/base/icons/assets/vender/other/mcp.svg b/web/app/components/base/icons/assets/vender/other/mcp.svg new file mode 100644 index 0000000000..7415c060dd --- /dev/null +++ b/web/app/components/base/icons/assets/vender/other/mcp.svg @@ -0,0 +1,4 @@ + + + + diff --git a/web/app/components/base/icons/assets/vender/other/no-tool-placeholder.svg b/web/app/components/base/icons/assets/vender/other/no-tool-placeholder.svg new file mode 100644 index 0000000000..8b8729412f --- /dev/null +++ b/web/app/components/base/icons/assets/vender/other/no-tool-placeholder.svg @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/vender/workflow/window-cursor.svg b/web/app/components/base/icons/assets/vender/workflow/window-cursor.svg new file mode 100644 index 0000000000..af8a9bac94 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/workflow/window-cursor.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/web/app/components/base/icons/src/vender/line/others/SearchMenu.json b/web/app/components/base/icons/src/vender/line/others/SearchMenu.json new file mode 100644 index 0000000000..5222574040 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/others/SearchMenu.json @@ -0,0 +1,77 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "32", + "height": "32", + "viewBox": "0 0 32 32", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M28.0049 16C28.0049 20.4183 24.4231 24 20.0049 24C15.5866 24 12.0049 20.4183 12.0049 16C12.0049 11.5817 15.5866 8 20.0049 8C24.4231 8 28.0049 11.5817 28.0049 16Z", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M4.00488 16H6.67155", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M4.00488 9.33334H8.00488", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M4.00488 22.6667H8.00488", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M26 22L29.3333 25.3333", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + }, + "name": "SearchMenu" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/others/SearchMenu.tsx b/web/app/components/base/icons/src/vender/line/others/SearchMenu.tsx new file mode 100644 index 0000000000..4826abb20f --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/others/SearchMenu.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './SearchMenu.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconData } from '@/app/components/base/icons/IconBase' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject>; + }, +) => + +Icon.displayName = 'SearchMenu' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/others/index.ts b/web/app/components/base/icons/src/vender/line/others/index.ts index 19d5f1ebb5..2322e9d9f1 100644 --- a/web/app/components/base/icons/src/vender/line/others/index.ts +++ b/web/app/components/base/icons/src/vender/line/others/index.ts @@ -9,4 +9,5 @@ export { default as GlobalVariable } from './GlobalVariable' export { default as Icon3Dots } from './Icon3Dots' export { default as LongArrowLeft } from './LongArrowLeft' export { default as LongArrowRight } from './LongArrowRight' +export { default as SearchMenu } from './SearchMenu' export { default as Tools } from './Tools' diff --git a/web/app/components/base/icons/src/vender/other/Mcp.json b/web/app/components/base/icons/src/vender/other/Mcp.json new file mode 100644 index 0000000000..7caa70b16b --- /dev/null +++ b/web/app/components/base/icons/src/vender/other/Mcp.json @@ -0,0 +1,35 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M9.20626 1.68651C9.61828 1.68651 10.014 1.8473 10.3093 2.13466C10.4536 2.27516 10.5684 2.44313 10.6468 2.62868C10.7252 2.81422 10.7657 3.01358 10.7659 3.21501C10.7661 3.41645 10.7259 3.61588 10.6478 3.80156C10.5697 3.98723 10.4552 4.1554 10.3111 4.29614L5.86656 8.65516C5.81837 8.70203 5.78006 8.75808 5.7539 8.82001C5.72775 8.88194 5.71427 8.94848 5.71427 9.01571C5.71427 9.08294 5.72775 9.14948 5.7539 9.21141C5.78006 9.27334 5.81837 9.32939 5.86656 9.37626C5.96503 9.47212 6.09703 9.52576 6.23445 9.52576C6.37187 9.52576 6.50387 9.47212 6.60234 9.37626L6.66222 9.31698L6.66345 9.31576L11.0463 5.01725C11.3417 4.73067 11.7372 4.57056 12.1488 4.5709C12.5604 4.57124 12.9556 4.73202 13.2506 5.01908L13.2811 5.04902C13.4256 5.18967 13.5405 5.35786 13.6189 5.54363C13.6973 5.72941 13.7377 5.92903 13.7377 6.13068C13.7377 6.33233 13.6973 6.53195 13.6189 6.71773C13.5405 6.9035 13.4256 7.07169 13.2811 7.21234L7.96082 12.43C7.84828 12.5393 7.75882 12.6701 7.69773 12.8147C7.63664 12.9592 7.60517 13.1145 7.60517 13.2714C7.60517 13.4284 7.63664 13.5837 7.69773 13.7282C7.75882 13.8728 7.84828 14.0036 7.96082 14.1129L9.05348 15.1842C9.15192 15.2799 9.28378 15.3334 9.42106 15.3334C9.55834 15.3334 9.6902 15.2799 9.78864 15.1842C9.83683 15.1373 9.87514 15.0813 9.9013 15.0194C9.92746 14.9574 9.94094 14.8909 9.94094 14.8237C9.94094 14.7564 9.92746 14.6899 9.9013 14.628C9.87514 14.566 9.83683 14.51 9.78864 14.4631L8.69598 13.3912C8.67992 13.3756 8.66716 13.357 8.65844 13.3363C8.64973 13.3157 8.64523 13.2935 8.64523 13.2711C8.64523 13.2488 8.64973 13.2266 8.65844 13.206C8.66716 13.1853 8.67992 13.1667 8.69598 13.1511L14.0163 7.93405C14.2572 7.69971 14.4488 7.41943 14.5796 7.10979C14.7104 6.80014 14.7778 6.46742 14.7778 6.13129C14.7778 5.79516 14.7104 5.46244 14.5796 5.1528C14.4488 4.84315 14.2572 4.56288 14.0163 4.32853L13.9857 4.29797C13.6978 4.01697 13.3493 3.80582 12.9669 3.6808C12.5845 3.55578 12.1785 3.52022 11.7802 3.57687C11.8371 3.1838 11.8001 2.78285 11.6722 2.40684C11.5443 2.03083 11.3292 1.69045 11.0445 1.41356C10.5524 0.93469 9.89288 0.666748 9.20626 0.666748C8.51964 0.666748 7.86012 0.93469 7.36805 1.41356L1.48555 7.18239C1.43735 7.22926 1.39905 7.28532 1.37289 7.34725C1.34673 7.40917 1.33325 7.47572 1.33325 7.54294C1.33325 7.61017 1.34673 7.67672 1.37289 7.73864C1.39905 7.80057 1.43735 7.85663 1.48555 7.9035C1.58399 7.99918 1.71585 8.0527 1.85313 8.0527C1.9904 8.0527 2.12227 7.99918 2.22071 7.9035L8.10321 2.13466C8.39848 1.8473 8.79424 1.68651 9.20626 1.68651Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M9.68688 3.41201C9.66072 3.47394 9.62241 3.52999 9.57422 3.57686L5.22314 7.8436C5.07864 7.98425 4.96378 8.15243 4.88535 8.33821C4.80693 8.52399 4.76652 8.7236 4.76652 8.92526C4.76652 9.12691 4.80693 9.32652 4.88535 9.5123C4.96378 9.69808 5.07864 9.86626 5.22314 10.0069C5.51841 10.2943 5.91417 10.4551 6.32619 10.4551C6.73821 10.4551 7.13397 10.2943 7.42924 10.0069L11.7797 5.74017C11.8782 5.64431 12.0102 5.59067 12.1476 5.59067C12.285 5.59067 12.417 5.64431 12.5155 5.74017C12.5637 5.78704 12.602 5.8431 12.6281 5.90503C12.6543 5.96696 12.6678 6.0335 12.6678 6.10073C12.6678 6.16795 12.6543 6.2345 12.6281 6.29643C12.602 6.35835 12.5637 6.41441 12.5155 6.46128L8.1644 10.728C7.67225 11.2067 7.01276 11.4746 6.32619 11.4746C5.63962 11.4746 4.98013 11.2067 4.48798 10.728C4.24701 10.4937 4.05547 10.2134 3.92468 9.90375C3.79389 9.59411 3.7265 9.26139 3.7265 8.92526C3.7265 8.58912 3.79389 8.2564 3.92468 7.94676C4.05547 7.63712 4.24701 7.35684 4.48798 7.1225L8.83845 2.85576C8.93691 2.75989 9.06891 2.70625 9.20633 2.70625C9.34375 2.70625 9.47575 2.75989 9.57422 2.85576C9.62241 2.90263 9.66072 2.95868 9.68688 3.02061C9.71304 3.08254 9.72651 3.14908 9.72651 3.21631C9.72651 3.28353 9.71304 3.35008 9.68688 3.41201Z", + "fill": "currentColor" + }, + "children": [] + } + ] + }, + "name": "Mcp" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/other/Mcp.tsx b/web/app/components/base/icons/src/vender/other/Mcp.tsx new file mode 100644 index 0000000000..00ffa4a831 --- /dev/null +++ b/web/app/components/base/icons/src/vender/other/Mcp.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Mcp.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconData } from '@/app/components/base/icons/IconBase' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject>; + }, +) => + +Icon.displayName = 'Mcp' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/other/NoToolPlaceholder.json b/web/app/components/base/icons/src/vender/other/NoToolPlaceholder.json new file mode 100644 index 0000000000..d33d62d344 --- /dev/null +++ b/web/app/components/base/icons/src/vender/other/NoToolPlaceholder.json @@ -0,0 +1,279 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "204", + "height": "36", + "viewBox": "0 0 204 36", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "opacity": "0.1" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M3.33333 18.3333C3.33333 18.5423 3.64067 18.9056 4.35365 19.2621C5.27603 19.7233 6.58451 20 8 20C9.41547 20 10.7239 19.7233 11.6463 19.2621C12.3593 18.9056 12.6667 18.5423 12.6667 18.3333V16.8858C11.5667 17.5655 9.88487 18 8 18C6.11515 18 4.43331 17.5655 3.33333 16.8858V18.3333ZM12.6667 20.2191C11.5667 20.8988 9.88487 21.3333 8 21.3333C6.11515 21.3333 4.43331 20.8988 3.33333 20.2191V21.6667C3.33333 21.8756 3.64067 22.2389 4.35365 22.5954C5.27603 23.0566 6.58451 23.3333 8 23.3333C9.41547 23.3333 10.7239 23.0566 11.6463 22.5954C12.3593 22.2389 12.6667 21.8756 12.6667 21.6667V20.2191ZM2 21.6667V15C2 13.3431 4.68629 12 8 12C11.3137 12 14 13.3431 14 15V21.6667C14 23.3235 11.3137 24.6667 8 24.6667C4.68629 24.6667 2 23.3235 2 21.6667ZM8 16.6667C9.41547 16.6667 10.7239 16.3899 11.6463 15.9288C12.3593 15.5723 12.6667 15.2089 12.6667 15C12.6667 14.7911 12.3593 14.4277 11.6463 14.0712C10.7239 13.6101 9.41547 13.3333 8 13.3333C6.58451 13.3333 5.27603 13.6101 4.35365 14.0712C3.64067 14.4277 3.33333 14.7911 3.33333 15C3.33333 15.2089 3.64067 15.5723 4.35365 15.9288C5.27603 16.3899 6.58451 16.6667 8 16.6667Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "opacity": "0.3" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M41.3337 11.3333C41.7019 11.3333 42.0003 11.6318 42.0003 12V15.3333C42.0003 15.7015 41.7019 16 41.3337 16H38.0003V24.6667C38.0003 25.0349 37.7019 25.3333 37.3337 25.3333H34.667C34.2988 25.3333 34.0003 25.0349 34.0003 24.6667V16H30.3337C29.9655 16 29.667 15.7015 29.667 15.3333V13.7454C29.667 13.4929 29.8097 13.262 30.0355 13.1491L33.667 11.3333H41.3337ZM38.0003 12.6667H33.9818L31.0003 14.1574V14.6667H35.3337V24H36.667V14.6667H38.0003V12.6667ZM40.667 12.6667H39.3337V14.6667H40.667V12.6667Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "opacity": "0.6" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M60.6667 13.3333C60.6667 11.8606 61.8606 10.6667 63.3333 10.6667C64.8061 10.6667 66 11.8606 66 13.3333H69.3333C69.7015 13.3333 70 13.6318 70 14V16.7805C70 16.9969 69.8949 17.1998 69.7183 17.3248C69.5415 17.4497 69.3152 17.4811 69.1112 17.409C68.973 17.3602 68.8237 17.3333 68.6667 17.3333C67.9303 17.3333 67.3333 17.9303 67.3333 18.6667C67.3333 19.4031 67.9303 20 68.6667 20C68.8237 20 68.973 19.9731 69.1112 19.9243C69.3152 19.8522 69.5415 19.8836 69.7183 20.0085C69.8949 20.1335 70 20.3365 70 20.5529V23.3333C70 23.7015 69.7015 24 69.3333 24H58.6667C58.2985 24 58 23.7015 58 23.3333V14C58 13.6318 58.2985 13.3333 58.6667 13.3333H60.6667ZM63.3333 12C62.597 12 62 12.5969 62 13.3333C62 13.4903 62.0269 13.6397 62.0757 13.7778C62.1478 13.9819 62.1164 14.2082 61.9915 14.3849C61.8665 14.5616 61.6635 14.6667 61.4471 14.6667H59.3333V22.6667H68.6667V21.3333C67.1939 21.3333 66 20.1394 66 18.6667C66 17.1939 67.1939 16 68.6667 16V14.6667H65.2195C65.0031 14.6667 64.8002 14.5616 64.6752 14.3849C64.5503 14.2082 64.5189 13.9819 64.591 13.7778C64.6398 13.6397 64.6667 13.4904 64.6667 13.3333C64.6667 12.5969 64.0697 12 63.3333 12Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "x": "84.5", + "y": "0.5", + "width": "35", + "height": "35", + "rx": "9.5", + "stroke": "#101828", + "stroke-opacity": "0.04" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M96.167 16.3333H107.834V25.5H96.167V16.3333Z", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M100.333 19.6667H103.666", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M107.833 12.1667L105.583 15.9167", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M97.4888 11.9517C97.5447 11.9238 97.5901 11.8784 97.6181 11.8224L97.9911 11.0765C98.0976 10.8634 98.4017 10.8634 98.5083 11.0765L98.8813 11.8224C98.9092 11.8784 98.9546 11.9238 99.0106 11.9517L99.7565 12.3247C99.9696 12.4313 99.9696 12.7354 99.7565 12.842L99.0106 13.2149C98.9546 13.2429 98.9092 13.2883 98.8813 13.3442L98.5083 14.0902C98.4017 14.3033 98.0976 14.3033 97.9911 14.0902L97.6181 13.3442C97.5901 13.2883 97.5447 13.2429 97.4888 13.2149L96.7429 12.842C96.5297 12.7354 96.5297 12.4313 96.7429 12.3247L97.4888 11.9517Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M101.882 10.5438C101.952 10.5089 102.009 10.4521 102.044 10.3822L102.51 9.4498C102.643 9.1834 103.023 9.1834 103.157 9.4498L103.623 10.3822C103.658 10.4521 103.714 10.5089 103.784 10.5438L104.717 11.0101C104.983 11.1432 104.983 11.5234 104.717 11.6566L103.784 12.1228C103.714 12.1578 103.658 12.2145 103.623 12.2845L103.157 13.2169C103.023 13.4833 102.643 13.4833 102.51 13.2169L102.044 12.2845C102.009 12.2145 101.952 12.1578 101.882 12.1228L100.95 11.6566C100.683 11.5234 100.683 11.1432 100.95 11.0101L101.882 10.5438Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "g", + "attributes": { + "opacity": "0.6", + "clip-path": "url(#clip0_9296_51042)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M145.809 14.7521L145.645 15.1292C145.525 15.4053 145.143 15.4053 145.022 15.1292L144.858 14.7521C144.565 14.0796 144.037 13.5443 143.379 13.2514L142.872 13.0261C142.599 12.9044 142.599 12.5059 142.872 12.3841L143.35 12.1714C144.026 11.871 144.563 11.3158 144.851 10.6206L145.02 10.213C145.138 9.92899 145.53 9.92899 145.647 10.213L145.816 10.6206C146.104 11.3158 146.641 11.871 147.317 12.1714L147.795 12.3841C148.069 12.5059 148.069 12.9044 147.795 13.0261L147.289 13.2514C146.63 13.5443 146.102 14.0796 145.809 14.7521ZM138 11.3333C140.712 11.3333 142.951 13.3571 143.289 15.9766L144.79 18.3358C144.889 18.4911 144.869 18.7231 144.64 18.8211L143.334 19.3807V21.3333C143.334 22.0697 142.737 22.6667 142 22.6667H140.668L140.667 24.6667H134.667L134.667 22.2041C134.667 21.4168 134.376 20.6725 133.837 20.0007C133.105 19.0875 132.667 17.9283 132.667 16.6667C132.667 13.7211 135.055 11.3333 138 11.3333ZM138 12.6667C135.791 12.6667 134 14.4575 134 16.6667C134 17.5899 134.312 18.4619 134.878 19.1666C135.607 20.076 136.001 21.1115 136 22.2042L136 23.3333H139.334L139.335 21.3333H142V18.5013L143.033 18.0587L142.005 16.4417L141.967 16.1475C141.711 14.1676 140.017 12.6667 138 12.6667ZM144.993 21.3286L146.103 22.0683C146.88 20.9041 147.334 19.5051 147.334 18.0001C147.334 17.5447 147.292 17.0991 147.213 16.6667L145.917 17C145.972 17.3252 146 17.6593 146 18.0001C146 19.2314 145.629 20.3761 144.993 21.3286Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "opacity": "0.3" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M167.149 16.3522L167.251 16.3822L174.224 19.0391L174.317 19.082C174.722 19.308 174.777 19.8792 174.424 20.179L174.341 20.2389L171.817 21.8171L170.239 24.3418C169.962 24.784 169.324 24.7511 169.082 24.3171L169.039 24.2246L166.382 17.2513C166.188 16.742 166.644 16.2421 167.149 16.3522ZM169.812 22.5085L170.767 20.9811L170.811 20.9186C170.858 20.859 170.916 20.8076 170.981 20.7669L172.508 19.8119L168.152 18.1524L169.812 22.5085Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M165.212 20.3978L163.562 22.0475L162.619 21.1048L164.269 19.4551L165.212 20.3978Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M163.666 18H161.333V16.6667H163.666V18Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M165.212 14.2689L164.269 15.2116L162.619 13.5619L163.562 12.6192L165.212 14.2689Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M172.047 13.5619L170.397 15.2116L169.455 14.2689L171.104 12.6192L172.047 13.5619Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M168 13.6667H166.666V11.3333H168V13.6667Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "opacity": "0.1" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M202.666 23.3333V14.6667L201.333 12H190.666L189.333 14.669V23.3333C189.333 23.7015 189.631 24 190 24H202C202.368 24 202.666 23.7015 202.666 23.3333ZM190.666 16H201.333V22.6667H190.666V16ZM191.49 13.3333H200.509L201.176 14.6667H190.824L191.49 13.3333ZM198 17.3333H194V18.6667H198V17.3333Z", + "fill": "currentColor", + "fill-opacity": "0.3" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "clipPath", + "attributes": { + "id": "clip0_9296_51042" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "16", + "height": "16", + "fill": "white", + "transform": "translate(132 10)" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "NoToolPlaceholder" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/other/NoToolPlaceholder.tsx b/web/app/components/base/icons/src/vender/other/NoToolPlaceholder.tsx new file mode 100644 index 0000000000..da8fddee22 --- /dev/null +++ b/web/app/components/base/icons/src/vender/other/NoToolPlaceholder.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './NoToolPlaceholder.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconData } from '@/app/components/base/icons/IconBase' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject>; + }, +) => + +Icon.displayName = 'NoToolPlaceholder' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/other/index.ts b/web/app/components/base/icons/src/vender/other/index.ts index 8ddf5e7a86..8a7bb7ae28 100644 --- a/web/app/components/base/icons/src/vender/other/index.ts +++ b/web/app/components/base/icons/src/vender/other/index.ts @@ -1,5 +1,7 @@ export { default as AnthropicText } from './AnthropicText' export { default as Generator } from './Generator' export { default as Group } from './Group' +export { default as Mcp } from './Mcp' +export { default as NoToolPlaceholder } from './NoToolPlaceholder' export { default as Openai } from './Openai' export { default as ReplayLine } from './ReplayLine' diff --git a/web/app/components/base/icons/src/vender/workflow/WindowCursor.json b/web/app/components/base/icons/src/vender/workflow/WindowCursor.json new file mode 100644 index 0000000000..b64ba912bb --- /dev/null +++ b/web/app/components/base/icons/src/vender/workflow/WindowCursor.json @@ -0,0 +1,62 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M1.33325 4.66663C1.33325 3.56206 2.22869 2.66663 3.33325 2.66663H12.6666C13.7712 2.66663 14.6666 3.56206 14.6666 4.66663V8.16663C14.6666 8.53483 14.3681 8.83329 13.9999 8.83329C13.6317 8.83329 13.3333 8.53483 13.3333 8.16663V4.66663C13.3333 4.29844 13.0348 3.99996 12.6666 3.99996H3.33325C2.96507 3.99996 2.66659 4.29844 2.66659 4.66663V12C2.66659 12.3682 2.96507 12.6666 3.33325 12.6666H7.99992C8.36812 12.6666 8.66658 12.9651 8.66658 13.3333C8.66658 13.7015 8.36812 14 7.99992 14H3.33325C2.22869 14 1.33325 13.1046 1.33325 12V4.66663Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M3.66659 5.83329C3.66659 6.29353 4.03968 6.66663 4.49992 6.66663C4.96016 6.66663 5.33325 6.29353 5.33325 5.83329C5.33325 5.37305 4.96016 4.99996 4.49992 4.99996C4.03968 4.99996 3.66659 5.37305 3.66659 5.83329Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M5.99992 5.83329C5.99992 6.29353 6.37301 6.66663 6.83325 6.66663C7.29352 6.66663 7.66658 6.29353 7.66658 5.83329C7.66658 5.37305 7.29352 4.99996 6.83325 4.99996C6.37301 4.99996 5.99992 5.37305 5.99992 5.83329Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M8.33325 5.83329C8.33325 6.29353 8.70632 6.66663 9.16658 6.66663C9.62685 6.66663 9.99992 6.29353 9.99992 5.83329C9.99992 5.37305 9.62685 4.99996 9.16658 4.99996C8.70632 4.99996 8.33325 5.37305 8.33325 5.83329Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M10.5293 9.69609C10.2933 9.62349 10.0365 9.68729 9.86185 9.86189C9.68725 10.0365 9.62345 10.2934 9.69605 10.5294L11.0294 14.8627C11.1095 15.1231 11.3401 15.3086 11.6116 15.331C11.8832 15.3535 12.1411 15.2085 12.2629 14.9648L13.1635 13.1636L14.9647 12.263C15.2085 12.1411 15.3535 11.8832 15.331 11.6116C15.3085 11.3401 15.1231 11.1096 14.8627 11.0294L10.5293 9.69609Z", + "fill": "currentColor" + }, + "children": [] + } + ] + }, + "name": "WindowCursor" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/workflow/WindowCursor.tsx b/web/app/components/base/icons/src/vender/workflow/WindowCursor.tsx new file mode 100644 index 0000000000..8f48dc0b14 --- /dev/null +++ b/web/app/components/base/icons/src/vender/workflow/WindowCursor.tsx @@ -0,0 +1,20 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './WindowCursor.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconData } from '@/app/components/base/icons/IconBase' + +const Icon = ( + { + ref, + ...props + }: React.SVGProps & { + ref?: React.RefObject>; + }, +) => + +Icon.displayName = 'WindowCursor' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/workflow/index.ts b/web/app/components/base/icons/src/vender/workflow/index.ts index 7167b71b44..61fbd4b21c 100644 --- a/web/app/components/base/icons/src/vender/workflow/index.ts +++ b/web/app/components/base/icons/src/vender/workflow/index.ts @@ -19,3 +19,4 @@ export { default as ParameterExtractor } from './ParameterExtractor' export { default as QuestionClassifier } from './QuestionClassifier' export { default as TemplatingTransform } from './TemplatingTransform' export { default as VariableX } from './VariableX' +export { default as WindowCursor } from './WindowCursor' diff --git a/web/app/components/base/prompt-editor/index.tsx b/web/app/components/base/prompt-editor/index.tsx index 94a65e4b62..a87a51cd50 100644 --- a/web/app/components/base/prompt-editor/index.tsx +++ b/web/app/components/base/prompt-editor/index.tsx @@ -64,8 +64,9 @@ import cn from '@/utils/classnames' export type PromptEditorProps = { instanceId?: string compact?: boolean + wrapperClassName?: string className?: string - placeholder?: string + placeholder?: string | JSX.Element placeholderClassName?: string style?: React.CSSProperties value?: string @@ -85,6 +86,7 @@ export type PromptEditorProps = { const PromptEditor: FC = ({ instanceId, compact, + wrapperClassName, className, placeholder, placeholderClassName, @@ -147,10 +149,25 @@ const PromptEditor: FC = ({ return ( -
+
} - placeholder={} + contentEditable={ + + } + placeholder={ + + } ErrorBoundary={LexicalErrorBoundary} /> { const { t } = useTranslation() diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx index 731841f423..da5ad84cb1 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx @@ -32,12 +32,14 @@ import Tooltip from '@/app/components/base/tooltip' import { isExceptionVariable } from '@/app/components/workflow/utils' import VarFullPathPanel from '@/app/components/workflow/nodes/_base/components/variable/var-full-path-panel' import { Type } from '@/app/components/workflow/nodes/llm/types' -import type { ValueSelector } from '@/app/components/workflow/types' +import type { ValueSelector, Var } from '@/app/components/workflow/types' type WorkflowVariableBlockComponentProps = { nodeKey: string variables: string[] workflowNodesMap: WorkflowNodesMap + environmentVariables?: Var[] + conversationVariables?: Var[] getVarType?: (payload: { nodeId: string, valueSelector: ValueSelector, @@ -49,6 +51,8 @@ const WorkflowVariableBlockComponent = ({ variables, workflowNodesMap = {}, getVarType, + environmentVariables, + conversationVariables, }: WorkflowVariableBlockComponentProps) => { const { t } = useTranslation() const [editor] = useLexicalComposerContext() @@ -68,6 +72,19 @@ const WorkflowVariableBlockComponent = ({ const isChatVar = isConversationVar(variables) const isException = isExceptionVariable(varName, node?.type) + let variableValid = true + if (isEnv) { + if (environmentVariables) + variableValid = environmentVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}`) + } + else if (isChatVar) { + if (conversationVariables) + variableValid = conversationVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}`) + } + else { + variableValid = !!node + } + const reactflow = useReactFlow() const store = useStoreApi() @@ -113,7 +130,7 @@ const WorkflowVariableBlockComponent = ({ className={cn( 'group/wrap relative mx-0.5 flex h-[18px] select-none items-center rounded-[5px] border pl-0.5 pr-[3px] hover:border-state-accent-solid hover:bg-state-accent-hover', isSelected ? ' border-state-accent-solid bg-state-accent-hover' : ' border-components-panel-border-subtle bg-components-badge-white-to-dark', - !node && !isEnv && !isChatVar && '!border-state-destructive-solid !bg-state-destructive-hover', + !variableValid && '!border-state-destructive-solid !bg-state-destructive-hover', )} onClick={(e) => { e.stopPropagation() @@ -156,7 +173,7 @@ const WorkflowVariableBlockComponent = ({ isException && 'text-text-warning', )} title={varName}>{varName}
{ - !node && !isEnv && !isChatVar && ( + !variableValid && ( ) } @@ -164,7 +181,7 @@ const WorkflowVariableBlockComponent = ({
) - if (!node && !isEnv && !isChatVar) { + if (!variableValid) { return ( {Item} diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx index dce636d92d..f828bdbc14 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx @@ -3,6 +3,7 @@ import { DecoratorNode } from 'lexical' import type { WorkflowVariableBlockType } from '../../types' import WorkflowVariableBlockComponent from './component' import type { GetVarType } from '../../types' +import type { Var } from '@/app/components/workflow/types' export type WorkflowNodesMap = WorkflowVariableBlockType['workflowNodesMap'] @@ -10,31 +11,37 @@ export type SerializedNode = SerializedLexicalNode & { variables: string[] workflowNodesMap: WorkflowNodesMap getVarType?: GetVarType + environmentVariables?: Var[] + conversationVariables?: Var[] } export class WorkflowVariableBlockNode extends DecoratorNode { __variables: string[] __workflowNodesMap: WorkflowNodesMap __getVarType?: GetVarType + __environmentVariables?: Var[] + __conversationVariables?: Var[] static getType(): string { return 'workflow-variable-block' } static clone(node: WorkflowVariableBlockNode): WorkflowVariableBlockNode { - return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key) + return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables) } isInline(): boolean { return true } - constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey) { + constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[]) { super(key) this.__variables = variables this.__workflowNodesMap = workflowNodesMap this.__getVarType = getVarType + this.__environmentVariables = environmentVariables + this.__conversationVariables = conversationVariables } createDOM(): HTMLElement { @@ -54,12 +61,14 @@ export class WorkflowVariableBlockNode extends DecoratorNode variables={this.__variables} workflowNodesMap={this.__workflowNodesMap} getVarType={this.__getVarType!} + environmentVariables={this.__environmentVariables} + conversationVariables={this.__conversationVariables} /> ) } static importJSON(serializedNode: SerializedNode): WorkflowVariableBlockNode { - const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType) + const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables) return node } @@ -71,6 +80,8 @@ export class WorkflowVariableBlockNode extends DecoratorNode variables: this.getVariables(), workflowNodesMap: this.getWorkflowNodesMap(), getVarType: this.getVarType(), + environmentVariables: this.getEnvironmentVariables(), + conversationVariables: this.getConversationVariables(), } } @@ -89,12 +100,22 @@ export class WorkflowVariableBlockNode extends DecoratorNode return self.__getVarType } + getEnvironmentVariables(): any { + const self = this.getLatest() + return self.__environmentVariables + } + + getConversationVariables(): any { + const self = this.getLatest() + return self.__conversationVariables + } + getTextContent(): string { return `{{#${this.getVariables().join('.')}#}}` } } -export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType): WorkflowVariableBlockNode { - return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType) +export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[]): WorkflowVariableBlockNode { + return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables) } export function $isWorkflowVariableBlockNode( diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx index 288008bbcc..4d0c80f10f 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx @@ -18,6 +18,7 @@ const WorkflowVariableBlockReplacementBlock = ({ workflowNodesMap, getVarType, onInsert, + variables, }: WorkflowVariableBlockType) => { const [editor] = useLexicalComposerContext() @@ -31,8 +32,8 @@ const WorkflowVariableBlockReplacementBlock = ({ onInsert() const nodePathString = textNode.getTextContent().slice(3, -3) - return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType)) - }, [onInsert, workflowNodesMap, getVarType]) + return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || [])) + }, [onInsert, workflowNodesMap, getVarType, variables]) const getMatch = useCallback((text: string) => { const matchArr = REGEX.exec(text) diff --git a/web/app/components/base/tooltip/index.tsx b/web/app/components/base/tooltip/index.tsx index 53f36be5fb..697d6e3d96 100644 --- a/web/app/components/base/tooltip/index.tsx +++ b/web/app/components/base/tooltip/index.tsx @@ -101,7 +101,7 @@ const Tooltip: FC = ({ > {popupContent && (
triggerMethod === 'hover' && setHoverPopup()} diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index b1e4087226..a1ff2f5d87 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -122,7 +122,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { return return ( -
+
{step === 1 && setShowModal(true) const modalCloseHandle = () => setShowModal(false) - const updateCurrentFile = (file: File) => { + const updateCurrentFile = useCallback((file: File) => { setCurrentFile(file) - } - const hideFilePreview = () => { + }, []) + + const hideFilePreview = useCallback(() => { setCurrentFile(undefined) - } + }, []) - const updateCurrentPage = (page: NotionPage) => { + const updateCurrentPage = useCallback((page: NotionPage) => { setCurrentNotionPage(page) - } + }, []) - const hideNotionPagePreview = () => { + const hideNotionPagePreview = useCallback(() => { setCurrentNotionPage(undefined) - } + }, []) + + const updateWebsite = useCallback((website: CrawlResultItem) => { + setCurrentWebsite(website) + }, []) - const hideWebsitePreview = () => { + const hideWebsitePreview = useCallback(() => { setCurrentWebsite(undefined) - } + }, []) const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type) const isInCreatePage = shouldShowDataSourceTypeList @@ -139,7 +144,7 @@ const StepOne = ({
{ shouldShowDataSourceTypeList && ( -
+
{t('datasetCreation.steps.one')}
) @@ -158,8 +163,8 @@ const StepOne = ({ if (dataSourceTypeDisable) return changeType(DataSourceType.FILE) - hideFilePreview() hideNotionPagePreview() + hideWebsitePreview() }} > @@ -182,7 +187,7 @@ const StepOne = ({ return changeType(DataSourceType.NOTION) hideFilePreview() - hideNotionPagePreview() + hideWebsitePreview() }} > @@ -201,7 +206,13 @@ const StepOne = ({ dataSourceType === DataSourceType.WEB && s.active, dataSourceTypeDisable && dataSourceType !== DataSourceType.WEB && s.disabled, )} - onClick={() => changeType(DataSourceType.WEB)} + onClick={() => { + if (dataSourceTypeDisable) + return + changeType(DataSourceType.WEB) + hideFilePreview() + hideNotionPagePreview() + }} >
export type TypeWithI18N = { @@ -19,6 +21,8 @@ export enum FormTypeEnum { toolSelector = 'tool-selector', multiToolSelector = 'array[tools]', appSelector = 'app-selector', + object = 'object', + array = 'array', dynamicSelect = 'dynamic-select', } @@ -109,6 +113,7 @@ export type FormShowOnObject = { } export type CredentialFormSchemaBase = { + name: string variable: string label: TypeWithI18N type: FormTypeEnum @@ -118,6 +123,7 @@ export type CredentialFormSchemaBase = { show_on: FormShowOnObject[] url?: string scope?: string + input_schema?: SchemaRoot } export type CredentialFormSchemaTextInput = CredentialFormSchemaBase & { diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/Form.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/Form.tsx index c5af4ed8a1..f1e3595d1e 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/Form.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/Form.tsx @@ -54,6 +54,7 @@ type FormProps< nodeId?: string nodeOutputVars?: NodeOutPutVar[], availableNodes?: Node[], + canChooseMCPTool?: boolean } function Form< @@ -79,6 +80,7 @@ function Form< nodeId, nodeOutputVars, availableNodes, + canChooseMCPTool, }: FormProps) { const language = useLanguage() const [changeKey, setChangeKey] = useState('') @@ -377,6 +379,7 @@ function Form< value={value[variable] || []} onChange={item => handleFormChange(variable, item as any)} supportCollapse + canChooseMCPTool={canChooseMCPTool} /> {fieldMoreInfo?.(formSchema)} {validating && changeKey === variable && } diff --git a/web/app/components/header/index.tsx b/web/app/components/header/index.tsx index 48973e50a8..fc511d2954 100644 --- a/web/app/components/header/index.tsx +++ b/web/app/components/header/index.tsx @@ -79,7 +79,7 @@ const Header = () => { } return ( -
+
{systemFeatures.branding.enabled && systemFeatures.branding.workspace_logo diff --git a/web/app/components/plugins/marketplace/search-box/index.tsx b/web/app/components/plugins/marketplace/search-box/index.tsx index 217007846c..5f19afbba6 100644 --- a/web/app/components/plugins/marketplace/search-box/index.tsx +++ b/web/app/components/plugins/marketplace/search-box/index.tsx @@ -1,8 +1,9 @@ 'use client' -import { RiCloseLine } from '@remixicon/react' +import { RiCloseLine, RiSearchLine } from '@remixicon/react' import TagsFilter from './tags-filter' import ActionButton from '@/app/components/base/action-button' import cn from '@/utils/classnames' +import { RiAddLine } from '@remixicon/react' type SearchBoxProps = { search: string @@ -13,6 +14,9 @@ type SearchBoxProps = { size?: 'small' | 'large' placeholder?: string locale?: string + supportAddCustomTool?: boolean + onShowAddCustomCollectionModal?: () => void + onAddedCustomTool?: () => void } const SearchBox = ({ search, @@ -23,46 +27,62 @@ const SearchBox = ({ size = 'small', placeholder = '', locale, + supportAddCustomTool, + onShowAddCustomCollectionModal, }: SearchBoxProps) => { return (
- -
-
-
- { - onSearchChange(e.target.value) - }} - placeholder={placeholder} - /> - { - search && ( -
- onSearchChange('')}> - - -
- ) - } +
+
+
+ + { + onSearchChange(e.target.value) + }} + placeholder={placeholder} + /> + { + search && ( +
+ onSearchChange('')}> + + +
+ ) + } +
+
+
+ {supportAddCustomTool && ( +
+ + + +
+ )}
) } diff --git a/web/app/components/plugins/marketplace/search-box/tags-filter.tsx b/web/app/components/plugins/marketplace/search-box/tags-filter.tsx index edf50dc874..bae6491727 100644 --- a/web/app/components/plugins/marketplace/search-box/tags-filter.tsx +++ b/web/app/components/plugins/marketplace/search-box/tags-filter.tsx @@ -2,9 +2,7 @@ import { useState } from 'react' import { - RiArrowDownSLine, - RiCloseCircleFill, - RiFilter3Line, + RiPriceTag3Line, } from '@remixicon/react' import { PortalToFollowElem, @@ -57,47 +55,15 @@ const TagsFilter = ({ onClick={() => setOpen(v => !v)} >
-
- +
+
-
- { - !selectedTagsLength && t('pluginTags.allTags') - } - { - !!selectedTagsLength && tags.map(tag => tagsMap[tag].label).slice(0, 2).join(',') - } - { - selectedTagsLength > 2 && ( -
- +{selectedTagsLength - 2} -
- ) - } -
- { - !!selectedTagsLength && ( - onTagsChange([])} - /> - ) - } - { - !selectedTagsLength && ( - - ) - }
diff --git a/web/app/components/plugins/plugin-detail-panel/multiple-tool-selector/index.tsx b/web/app/components/plugins/plugin-detail-panel/multiple-tool-selector/index.tsx index fef79644cd..2c700c6dc8 100644 --- a/web/app/components/plugins/plugin-detail-panel/multiple-tool-selector/index.tsx +++ b/web/app/components/plugins/plugin-detail-panel/multiple-tool-selector/index.tsx @@ -13,6 +13,7 @@ import type { Node } from 'reactflow' import type { NodeOutPutVar } from '@/app/components/workflow/types' import cn from '@/utils/classnames' import { ArrowDownRoundFill } from '@/app/components/base/icons/src/vender/solid/general' +import { useAllMCPTools } from '@/service/use-tools' type Props = { disabled?: boolean @@ -26,6 +27,7 @@ type Props = { nodeOutputVars: NodeOutPutVar[], availableNodes: Node[], nodeId?: string + canChooseMCPTool?: boolean } const MultipleToolSelector = ({ @@ -40,9 +42,16 @@ const MultipleToolSelector = ({ nodeOutputVars, availableNodes, nodeId, + canChooseMCPTool, }: Props) => { const { t } = useTranslation() - const enabledCount = value.filter(item => item.enabled).length + const { data: mcpTools } = useAllMCPTools() + const enabledCount = value.filter((item) => { + const isMCPTool = mcpTools?.find(tool => tool.id === item.provider_name) + if(isMCPTool) + return item.enabled && canChooseMCPTool + return item.enabled + }).length // collapse control const [collapse, setCollapse] = React.useState(false) const handleCollapse = () => { @@ -66,6 +75,19 @@ const MultipleToolSelector = ({ setOpen(false) } + const handleAddMultiple = (val: ToolValue[]) => { + const newValue = [...value, ...val] + // deduplication + const deduplication = newValue.reduce((acc, cur) => { + if (!acc.find(item => item.provider_name === cur.provider_name && item.tool_name === cur.tool_name)) + acc.push(cur) + return acc + }, [] as ToolValue[]) + // update value + onChange(deduplication) + setOpen(false) + } + // delete tool const handleDelete = (index: number) => { const newValue = [...value] @@ -140,8 +162,10 @@ const MultipleToolSelector = ({ value={item} selectedTools={value} onSelect={item => handleConfigure(item, index)} + onSelectMultiple={handleAddMultiple} onDelete={() => handleDelete(index)} supportEnableSwitch + canChooseMCPTool={canChooseMCPTool} isEdit />
@@ -164,6 +188,8 @@ const MultipleToolSelector = ({ panelShowState={panelShowState} onPanelShowStateChange={setPanelShowState} isEdit={false} + canChooseMCPTool={canChooseMCPTool} + onSelectMultiple={handleAddMultiple} /> ) diff --git a/web/app/components/plugins/plugin-detail-panel/tool-selector/index.tsx b/web/app/components/plugins/plugin-detail-panel/tool-selector/index.tsx index 350fe50933..15401f1057 100644 --- a/web/app/components/plugins/plugin-detail-panel/tool-selector/index.tsx +++ b/web/app/components/plugins/plugin-detail-panel/tool-selector/index.tsx @@ -5,7 +5,6 @@ import { useTranslation } from 'react-i18next' import Link from 'next/link' import { RiArrowLeftLine, - RiArrowRightUpLine, } from '@remixicon/react' import { PortalToFollowElem, @@ -15,6 +14,7 @@ import { import ToolTrigger from '@/app/components/plugins/plugin-detail-panel/tool-selector/tool-trigger' import ToolItem from '@/app/components/plugins/plugin-detail-panel/tool-selector/tool-item' import ToolPicker from '@/app/components/workflow/block-selector/tool-picker' +import ToolForm from '@/app/components/workflow/nodes/tool/components/tool-form' import Button from '@/app/components/base/button' import Indicator from '@/app/components/header/indicator' import ToolCredentialForm from '@/app/components/plugins/plugin-detail-panel/tool-selector/tool-credentials-form' @@ -23,13 +23,13 @@ import Textarea from '@/app/components/base/textarea' import Divider from '@/app/components/base/divider' import TabSlider from '@/app/components/base/tab-slider-plain' import ReasoningConfigForm from '@/app/components/plugins/plugin-detail-panel/tool-selector/reasoning-config-form' -import Form from '@/app/components/header/account-setting/model-provider-page/model-modal/Form' import { generateFormValue, getPlainValue, getStructureValue, toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' import { useAppContext } from '@/context/app-context' import { useAllBuiltInTools, useAllCustomTools, + useAllMCPTools, useAllWorkflowTools, useInvalidateAllBuiltInTools, useUpdateProviderCredentials, @@ -54,15 +54,9 @@ type Props = { scope?: string value?: ToolValue selectedTools?: ToolValue[] + onSelect: (tool: ToolValue) => void + onSelectMultiple?: (tool: ToolValue[]) => void isEdit?: boolean - onSelect: (tool: { - provider_name: string - tool_name: string - tool_label: string - settings?: Record - parameters?: Record - extra?: Record - }) => void onDelete?: () => void supportEnableSwitch?: boolean supportAddCustomTool?: boolean @@ -74,6 +68,7 @@ type Props = { nodeOutputVars: NodeOutPutVar[], availableNodes: Node[], nodeId?: string, + canChooseMCPTool?: boolean, } const ToolSelector: FC = ({ value, @@ -83,6 +78,7 @@ const ToolSelector: FC = ({ placement = 'left', offset = 4, onSelect, + onSelectMultiple, onDelete, scope, supportEnableSwitch, @@ -94,6 +90,7 @@ const ToolSelector: FC = ({ nodeOutputVars, availableNodes, nodeId = '', + canChooseMCPTool, }) => { const { t } = useTranslation() const [isShow, onShowChange] = useState(false) @@ -105,6 +102,7 @@ const ToolSelector: FC = ({ const { data: buildInTools } = useAllBuiltInTools() const { data: customTools } = useAllCustomTools() const { data: workflowTools } = useAllWorkflowTools() + const { data: mcpTools } = useAllMCPTools() const invalidateAllBuiltinTools = useInvalidateAllBuiltInTools() const invalidateInstalledPluginList = useInvalidateInstalledPluginList() @@ -112,18 +110,19 @@ const ToolSelector: FC = ({ const { inMarketPlace, manifest } = usePluginInstalledCheck(value?.provider_name) const currentProvider = useMemo(() => { - const mergedTools = [...(buildInTools || []), ...(customTools || []), ...(workflowTools || [])] + const mergedTools = [...(buildInTools || []), ...(customTools || []), ...(workflowTools || []), ...(mcpTools || [])] return mergedTools.find((toolWithProvider) => { return toolWithProvider.id === value?.provider_name }) - }, [value, buildInTools, customTools, workflowTools]) + }, [value, buildInTools, customTools, workflowTools, mcpTools]) const [isShowChooseTool, setIsShowChooseTool] = useState(false) - const handleSelectTool = (tool: ToolDefaultValue) => { + const getToolValue = (tool: ToolDefaultValue) => { const settingValues = generateFormValue(tool.params, toolParametersToFormSchemas(tool.paramSchemas.filter(param => param.form !== 'llm') as any)) const paramValues = generateFormValue(tool.params, toolParametersToFormSchemas(tool.paramSchemas.filter(param => param.form === 'llm') as any), true) - const toolValue = { + return { provider_name: tool.provider_id, + provider_show_name: tool.provider_name, type: tool.provider_type, tool_name: tool.tool_name, tool_label: tool.tool_label, @@ -136,9 +135,16 @@ const ToolSelector: FC = ({ }, schemas: tool.paramSchemas, } + } + const handleSelectTool = (tool: ToolDefaultValue) => { + const toolValue = getToolValue(tool) onSelect(toolValue) // setIsShowChooseTool(false) } + const handleSelectMultipleTool = (tool: ToolDefaultValue[]) => { + const toolValues = tool.map(item => getToolValue(item)) + onSelectMultiple?.(toolValues) + } const handleDescriptionChange = (e: React.ChangeEvent) => { onSelect({ @@ -169,7 +175,6 @@ const ToolSelector: FC = ({ const handleSettingsFormChange = (v: Record) => { const newValue = getStructureValue(v) - const toolValue = { ...value, settings: newValue, @@ -250,7 +255,9 @@ const ToolSelector: FC = ({ = ({

} + canChooseMCPTool={canChooseMCPTool} /> )} @@ -285,7 +293,6 @@ const ToolSelector: FC = ({
{t('plugin.detailPanel.toolSelector.toolLabel')}
= ({ disabled={false} supportAddCustomTool onSelect={handleSelectTool} + onSelectMultiple={handleSelectMultipleTool} scope={scope} selectedTools={selectedTools} + canChooseMCPTool={canChooseMCPTool} />
@@ -390,24 +399,13 @@ const ToolSelector: FC = ({ {/* user settings form */} {(currType === 'settings' || userSettingsOnly) && (
-
item.url - ? ( - {t('tools.howToGet')} - - ) - : null} />
)} diff --git a/web/app/components/plugins/plugin-detail-panel/tool-selector/reasoning-config-form.tsx b/web/app/components/plugins/plugin-detail-panel/tool-selector/reasoning-config-form.tsx index 750a8cfff6..98ad490348 100644 --- a/web/app/components/plugins/plugin-detail-panel/tool-selector/reasoning-config-form.tsx +++ b/web/app/components/plugins/plugin-detail-panel/tool-selector/reasoning-config-form.tsx @@ -3,25 +3,34 @@ import { useTranslation } from 'react-i18next' import produce from 'immer' import { RiArrowRightUpLine, + RiBracesLine, } from '@remixicon/react' import Tooltip from '@/app/components/base/tooltip' import Switch from '@/app/components/base/switch' -import Input from '@/app/components/workflow/nodes/_base/components/input-support-select-var' +import MixedVariableTextInput from '@/app/components/workflow/nodes/tool/components/mixed-variable-text-input' +import Input from '@/app/components/base/input' +import FormInputTypeSwitch from '@/app/components/workflow/nodes/_base/components/form-input-type-switch' +import FormInputBoolean from '@/app/components/workflow/nodes/_base/components/form-input-boolean' +import { SimpleSelect } from '@/app/components/base/select' +import CodeEditor from '@/app/components/workflow/nodes/_base/components/editor/code-editor' import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker' import AppSelector from '@/app/components/plugins/plugin-detail-panel/app-selector' import ModelParameterModal from '@/app/components/plugins/plugin-detail-panel/model-selector' +import { CodeLanguage } from '@/app/components/workflow/nodes/code/types' import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks' import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' import type { Node } from 'reactflow' import type { NodeOutPutVar, ValueSelector, - Var, } from '@/app/components/workflow/types' import type { ToolVarInputs } from '@/app/components/workflow/nodes/tool/types' import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' import { VarType } from '@/app/components/workflow/types' import cn from '@/utils/classnames' +import { useBoolean } from 'ahooks' +import SchemaModal from './schema-modal' +import type { SchemaRoot } from '@/app/components/workflow/nodes/llm/types' type Props = { value: Record @@ -42,73 +51,46 @@ const ReasoningConfigForm: React.FC = ({ }) => { const { t } = useTranslation() const language = useLanguage() - const handleAutomatic = (key: string, val: any) => { + const getVarKindType = (type: FormTypeEnum) => { + if (type === FormTypeEnum.file || type === FormTypeEnum.files) + return VarKindType.variable + if (type === FormTypeEnum.select || type === FormTypeEnum.boolean || type === FormTypeEnum.textNumber || type === FormTypeEnum.array || type === FormTypeEnum.object) + return VarKindType.constant + if (type === FormTypeEnum.textInput || type === FormTypeEnum.secretInput) + return VarKindType.mixed + } + + const handleAutomatic = (key: string, val: any, type: FormTypeEnum) => { onChange({ ...value, [key]: { - value: val ? null : value[key]?.value, + value: val ? null : { type: getVarKindType(type), value: null }, auto: val ? 1 : 0, }, }) } - - const [inputsIsFocus, setInputsIsFocus] = useState>({}) - const handleInputFocus = useCallback((variable: string) => { - return (value: boolean) => { - setInputsIsFocus((prev) => { - return { - ...prev, - [variable]: value, - } - }) - } - }, []) - const handleNotMixedTypeChange = useCallback((variable: string) => { - return (varValue: ValueSelector | string, varKindType: VarKindType) => { - const newValue = produce(value, (draft: ToolVarInputs) => { - const target = draft[variable].value - if (target) { - target.type = varKindType - target.value = varValue - } - else { - draft[variable].value = { - type: varKindType, - value: varValue, - } - } - }) - onChange(newValue) - } - }, [value, onChange]) - const handleMixedTypeChange = useCallback((variable: string) => { - return (itemValue: string) => { - const newValue = produce(value, (draft: ToolVarInputs) => { - const target = draft[variable].value - if (target) { - target.value = itemValue - } - else { - draft[variable].value = { - type: VarKindType.mixed, - value: itemValue, - } + const handleTypeChange = useCallback((variable: string, defaultValue: any) => { + return (newType: VarKindType) => { + const res = produce(value, (draft: ToolVarInputs) => { + draft[variable].value = { + type: newType, + value: newType === VarKindType.variable ? '' : defaultValue, } }) - onChange(newValue) + onChange(res) } - }, [value, onChange]) - const handleFileChange = useCallback((variable: string) => { - return (varValue: ValueSelector | string) => { - const newValue = produce(value, (draft: ToolVarInputs) => { + }, [onChange, value]) + const handleValueChange = useCallback((variable: string, varType: FormTypeEnum) => { + return (newValue: any) => { + const res = produce(value, (draft: ToolVarInputs) => { draft[variable].value = { - type: VarKindType.variable, - value: varValue, + type: getVarKindType(varType), + value: newValue, } }) - onChange(newValue) + onChange(res) } - }, [value, onChange]) + }, [onChange, value]) const handleAppChange = useCallback((variable: string) => { return (app: { app_id: string @@ -132,9 +114,29 @@ const ReasoningConfigForm: React.FC = ({ onChange(newValue) } }, [onChange, value]) + const handleVariableSelectorChange = useCallback((variable: string) => { + return (newValue: ValueSelector | string) => { + const res = produce(value, (draft: ToolVarInputs) => { + draft[variable].value = { + type: VarKindType.variable, + value: newValue, + } + }) + onChange(res) + } + }, [onChange, value]) - const renderField = (schema: any) => { + const [isShowSchema, { + setTrue: showSchema, + setFalse: hideSchema, + }] = useBoolean(false) + + const [schema, setSchema] = useState(null) + const [schemaRootName, setSchemaRootName] = useState('') + + const renderField = (schema: any, showSchema: (schema: SchemaRoot, rootName: string) => void) => { const { + default: defaultValue, variable, label, required, @@ -142,6 +144,9 @@ const ReasoningConfigForm: React.FC = ({ type, scope, url, + input_schema, + placeholder, + options, } = schema const auto = value[variable]?.auto const tooltipContent = (tooltip && ( @@ -149,89 +154,150 @@ const ReasoningConfigForm: React.FC = ({ popupContent={
{tooltip[language] || tooltip.en_US}
} - triggerClassName='ml-1 w-4 h-4' + triggerClassName='ml-0.5 w-4 h-4' asChild={false} /> )) const varInput = value[variable].value + const isString = type === FormTypeEnum.textInput || type === FormTypeEnum.secretInput const isNumber = type === FormTypeEnum.textNumber - const isSelect = type === FormTypeEnum.select + const isObject = type === FormTypeEnum.object + const isArray = type === FormTypeEnum.array + const isShowJSONEditor = isObject || isArray const isFile = type === FormTypeEnum.file || type === FormTypeEnum.files + const isBoolean = type === FormTypeEnum.boolean + const isSelect = type === FormTypeEnum.select const isAppSelector = type === FormTypeEnum.appSelector const isModelSelector = type === FormTypeEnum.modelSelector - // const isToolSelector = type === FormTypeEnum.toolSelector - const isString = !isNumber && !isSelect && !isFile && !isAppSelector && !isModelSelector + const showTypeSwitch = isNumber || isObject || isArray + const isConstant = varInput?.type === VarKindType.constant || !varInput?.type + const showVariableSelector = isFile || varInput?.type === VarKindType.variable + const targetVarType = () => { + if (isString) + return VarType.string + else if (isNumber) + return VarType.number + else if (type === FormTypeEnum.files) + return VarType.arrayFile + else if (type === FormTypeEnum.file) + return VarType.file + else if (isBoolean) + return VarType.boolean + else if (isObject) + return VarType.object + else if (isArray) + return VarType.arrayObject + else + return VarType.string + } + const getFilterVar = () => { + if (isNumber) + return (varPayload: any) => varPayload.type === VarType.number + else if (isString) + return (varPayload: any) => [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) + else if (isFile) + return (varPayload: any) => [VarType.file, VarType.arrayFile].includes(varPayload.type) + else if (isBoolean) + return (varPayload: any) => varPayload.type === VarType.boolean + else if (isObject) + return (varPayload: any) => varPayload.type === VarType.object + else if (isArray) + return (varPayload: any) => [VarType.array, VarType.arrayString, VarType.arrayNumber, VarType.arrayObject].includes(varPayload.type) + return undefined + } + return ( -
+
-
- {label[language] || label.en_US} +
+ {label[language] || label.en_US} {required && ( * )} {tooltipContent} + · + {targetVarType()} + {isShowJSONEditor && ( + + {t('workflow.nodes.agent.clickToViewParameterSchema')} +
} + asChild={false}> +
showSchema(input_schema as SchemaRoot, label[language] || label.en_US)} + > + +
+ + )} +
-
handleAutomatic(variable, !auto)}> +
handleAutomatic(variable, !auto, type)}> {t('plugin.detailPanel.toolSelector.auto')} handleAutomatic(variable, val)} + onChange={val => handleAutomatic(variable, val, type)} />
{auto === 0 && ( - <> +
+ {showTypeSwitch && ( + + )} {isString && ( - )} - {/* {isString && ( - varPayload.type === VarType.number || varPayload.type === VarType.secret || varPayload.type === VarType.string} + onChange={handleValueChange(variable, type)} + placeholder={placeholder?.[language] || placeholder?.en_US} /> - )} */} - {(isNumber || isSelect) && ( - varPayload.type === schema._type : undefined} - availableVars={isSelect ? nodeOutputVars : undefined} - schema={schema} + )} + {isBoolean && ( + )} - {isFile && ( - varPayload.type === VarType.file || varPayload.type === VarType.arrayFile} + {isSelect && ( + { + if (option.show_on.length) + return option.show_on.every(showOnItem => value[showOnItem.variable] === showOnItem.value) + + return true + }).map((option: { value: any; label: { [x: string]: any; en_US: any } }) => ({ value: option.value, name: option.label[language] || option.label.en_US }))} + onSelect={item => handleValueChange(variable, type)(item.value as string)} + placeholder={placeholder?.[language] || placeholder?.en_US} /> )} + {isShowJSONEditor && isConstant && ( +
+ {placeholder?.[language] || placeholder?.en_US}
} + /> +
+ )} {isAppSelector && ( = ({ scope={scope} /> )} - + {showVariableSelector && ( + + )} +
)} {url && ( = ({ } return (
- {schemas.map(schema => renderField(schema))} + {!isShowSchema && schemas.map(schema => renderField(schema, (s: SchemaRoot, rootName: string) => { + setSchema(s) + setSchemaRootName(rootName) + showSchema() + }))} + {isShowSchema && ( + + )}
) } diff --git a/web/app/components/plugins/plugin-detail-panel/tool-selector/schema-modal.tsx b/web/app/components/plugins/plugin-detail-panel/tool-selector/schema-modal.tsx new file mode 100644 index 0000000000..cd4cf71bac --- /dev/null +++ b/web/app/components/plugins/plugin-detail-panel/tool-selector/schema-modal.tsx @@ -0,0 +1,59 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import Modal from '@/app/components/base/modal' +import VisualEditor from '@/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor' +import type { SchemaRoot } from '@/app/components/workflow/nodes/llm/types' +import { MittProvider, VisualEditorContextProvider } from '@/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/context' +import { useTranslation } from 'react-i18next' +import { RiCloseLine } from '@remixicon/react' + +type Props = { + isShow: boolean + schema: SchemaRoot + rootName: string + onClose: () => void +} + +const SchemaModal: FC = ({ + isShow, + schema, + rootName, + onClose, +}) => { + const { t } = useTranslation() + return ( + +
+ {/* Header */} +
+
+ {t('workflow.nodes.agent.parameterSchema')} +
+
+ +
+
+ {/* Content */} +
+ + + + + +
+
+ + ) +} +export default React.memo(SchemaModal) diff --git a/web/app/components/plugins/plugin-detail-panel/tool-selector/tool-item.tsx b/web/app/components/plugins/plugin-detail-panel/tool-selector/tool-item.tsx index d74fccf968..5cc9b7a3a8 100644 --- a/web/app/components/plugins/plugin-detail-panel/tool-selector/tool-item.tsx +++ b/web/app/components/plugins/plugin-detail-panel/tool-selector/tool-item.tsx @@ -17,10 +17,13 @@ import { ToolTipContent } from '@/app/components/base/tooltip/content' import { InstallPluginButton } from '@/app/components/workflow/nodes/_base/components/install-plugin-button' import { SwitchPluginVersion } from '@/app/components/workflow/nodes/_base/components/switch-plugin-version' import cn from '@/utils/classnames' +import McpToolNotSupportTooltip from '@/app/components/workflow/nodes/_base/components/mcp-tool-not-support-tooltip' type Props = { icon?: any providerName?: string + isMCPTool?: boolean + providerShowName?: string toolLabel?: string showSwitch?: boolean switchValue?: boolean @@ -35,11 +38,14 @@ type Props = { onInstall?: () => void versionMismatch?: boolean open: boolean + canChooseMCPTool?: boolean, } const ToolItem = ({ open, icon, + isMCPTool, + providerShowName, providerName, toolLabel, showSwitch, @@ -54,11 +60,13 @@ const ToolItem = ({ isError, errorTip, versionMismatch, + canChooseMCPTool, }: Props) => { const { t } = useTranslation() - const providerNameText = providerName?.split('/').pop() + const providerNameText = isMCPTool ? providerShowName : providerName?.split('/').pop() const isTransparent = uninstalled || versionMismatch || isError const [isDeleting, setIsDeleting] = useState(false) + const isShowCanNotChooseMCPTip = isMCPTool && !canChooseMCPTool return (
{icon && ( -
+
{typeof icon === 'string' &&
} {typeof icon !== 'string' && }
@@ -75,18 +83,19 @@ const ToolItem = ({ {!icon && (
)} -
+
{providerNameText}
{toolLabel}
- {!noAuth && !isError && !uninstalled && !versionMismatch && ( + {!noAuth && !isError && !uninstalled && !versionMismatch && !isShowCanNotChooseMCPTip && ( @@ -103,7 +112,7 @@ const ToolItem = ({
- {!isError && !uninstalled && !noAuth && !versionMismatch && showSwitch && ( + {!isError && !uninstalled && !noAuth && !versionMismatch && !isShowCanNotChooseMCPTip && showSwitch && (
e.stopPropagation()}>
)} + {isShowCanNotChooseMCPTip && ( + + )} {!isError && !uninstalled && !versionMismatch && noAuth && (
+
+
{t('tools.createTool.authMethod.value')}
+ setTempCredential({ ...tempCredential, api_key_value: e.target.value })} + placeholder={t('tools.createTool.authMethod.types.apiValuePlaceholder')!} + /> +
+ )}
diff --git a/web/app/components/tools/edit-custom-collection-modal/modal.tsx b/web/app/components/tools/edit-custom-collection-modal/modal.tsx index 190c72790e..ce7ba8a735 100644 --- a/web/app/components/tools/edit-custom-collection-modal/modal.tsx +++ b/web/app/components/tools/edit-custom-collection-modal/modal.tsx @@ -184,6 +184,7 @@ const EditCustomCollectionModal: FC = ({ onClose={onHide} closable className='!h-[calc(100vh-16px)] !max-w-[630px] !p-0' + wrapperClassName='z-[1000]' >
diff --git a/web/app/components/tools/mcp/create-card.tsx b/web/app/components/tools/mcp/create-card.tsx new file mode 100644 index 0000000000..7416f85a2f --- /dev/null +++ b/web/app/components/tools/mcp/create-card.tsx @@ -0,0 +1,75 @@ +'use client' +import { useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useContext } from 'use-context-selector' +import { + RiAddCircleFill, + RiArrowRightUpLine, + RiBookOpenLine, +} from '@remixicon/react' +import MCPModal from './modal' +import I18n from '@/context/i18n' +import { getLanguage } from '@/i18n/language' +import { useAppContext } from '@/context/app-context' +import { useCreateMCP } from '@/service/use-tools' +import type { ToolWithProvider } from '@/app/components/workflow/types' + +type Props = { + handleCreate: (provider: ToolWithProvider) => void +} + +const NewMCPCard = ({ handleCreate }: Props) => { + const { t } = useTranslation() + const { locale } = useContext(I18n) + const language = getLanguage(locale) + const { isCurrentWorkspaceManager } = useAppContext() + + const { mutateAsync: createMCP } = useCreateMCP() + + const create = async (info: any) => { + const provider = await createMCP(info) + handleCreate(provider) + } + + const linkUrl = useMemo(() => { + if (language.startsWith('zh_')) + return 'https://docs.dify.ai/zh-hans/guides/tools/mcp' + if (language.startsWith('ja_jp')) + return 'https://docs.dify.ai/ja_jp/guides/tools/mcp' + return 'https://docs.dify.ai/en/guides/tools/mcp' + }, [language]) + + const [showModal, setShowModal] = useState(false) + + return ( + <> + {isCurrentWorkspaceManager && ( + + )} + {showModal && ( + setShowModal(false)} + /> + )} + + ) +} +export default NewMCPCard diff --git a/web/app/components/tools/mcp/detail/content.tsx b/web/app/components/tools/mcp/detail/content.tsx new file mode 100644 index 0000000000..3f927b990d --- /dev/null +++ b/web/app/components/tools/mcp/detail/content.tsx @@ -0,0 +1,308 @@ +'use client' +import React, { useCallback, useEffect } from 'react' +import type { FC } from 'react' +import { useBoolean } from 'ahooks' +import copy from 'copy-to-clipboard' +import { useTranslation } from 'react-i18next' +import { useAppContext } from '@/context/app-context' +import { + RiCloseLine, + RiLoader2Line, + RiLoopLeftLine, +} from '@remixicon/react' +import type { ToolWithProvider } from '../../../workflow/types' +import Icon from '@/app/components/plugins/card/base/card-icon' +import ActionButton from '@/app/components/base/action-button' +import Button from '@/app/components/base/button' +import Confirm from '@/app/components/base/confirm' +import Indicator from '@/app/components/header/indicator' +import Tooltip from '@/app/components/base/tooltip' +import MCPModal from '../modal' +import OperationDropdown from './operation-dropdown' +import ListLoading from './list-loading' +import ToolItem from './tool-item' +import { + useAuthorizeMCP, + useDeleteMCP, + useInvalidateMCPTools, + useMCPTools, + useUpdateMCP, + useUpdateMCPTools, +} from '@/service/use-tools' +import { openOAuthPopup } from '@/hooks/use-oauth' +import cn from '@/utils/classnames' + +type Props = { + detail: ToolWithProvider + onUpdate: (isDelete?: boolean) => void + onHide: () => void + isTriggerAuthorize: boolean + onFirstCreate: () => void +} + +const MCPDetailContent: FC = ({ + detail, + onUpdate, + onHide, + isTriggerAuthorize, + onFirstCreate, +}) => { + const { t } = useTranslation() + const { isCurrentWorkspaceManager } = useAppContext() + + const { data, isFetching: isGettingTools } = useMCPTools(detail.is_team_authorization ? detail.id : '') + const invalidateMCPTools = useInvalidateMCPTools() + const { mutateAsync: updateTools, isPending: isUpdating } = useUpdateMCPTools() + const { mutateAsync: authorizeMcp, isPending: isAuthorizing } = useAuthorizeMCP() + const toolList = data?.tools || [] + + const [isShowUpdateConfirm, { + setTrue: showUpdateConfirm, + setFalse: hideUpdateConfirm, + }] = useBoolean(false) + + const handleUpdateTools = useCallback(async () => { + hideUpdateConfirm() + if (!detail) + return + await updateTools(detail.id) + invalidateMCPTools(detail.id) + onUpdate() + }, [detail, hideUpdateConfirm, invalidateMCPTools, onUpdate, updateTools]) + + const { mutateAsync: updateMCP } = useUpdateMCP({}) + const { mutateAsync: deleteMCP } = useDeleteMCP({}) + + const [isShowUpdateModal, { + setTrue: showUpdateModal, + setFalse: hideUpdateModal, + }] = useBoolean(false) + + const [isShowDeleteConfirm, { + setTrue: showDeleteConfirm, + setFalse: hideDeleteConfirm, + }] = useBoolean(false) + + const [deleting, { + setTrue: showDeleting, + setFalse: hideDeleting, + }] = useBoolean(false) + + const handleOAuthCallback = useCallback(() => { + if (!isCurrentWorkspaceManager) + return + if (!detail.id) + return + handleUpdateTools() + }, [detail.id, handleUpdateTools, isCurrentWorkspaceManager]) + + const handleAuthorize = useCallback(async () => { + onFirstCreate() + if (!isCurrentWorkspaceManager) + return + if (!detail) + return + const res = await authorizeMcp({ + provider_id: detail.id, + }) + if (res.result === 'success') + handleUpdateTools() + + else if (res.authorization_url) + openOAuthPopup(res.authorization_url, handleOAuthCallback) + }, [onFirstCreate, isCurrentWorkspaceManager, detail, authorizeMcp, handleUpdateTools, handleOAuthCallback]) + + const handleUpdate = useCallback(async (data: any) => { + if (!detail) + return + const res = await updateMCP({ + ...data, + provider_id: detail.id, + }) + if ((res as any)?.result === 'success') { + hideUpdateModal() + onUpdate() + handleAuthorize() + } + }, [detail, updateMCP, hideUpdateModal, onUpdate, handleAuthorize]) + + const handleDelete = useCallback(async () => { + if (!detail) + return + showDeleting() + const res = await deleteMCP(detail.id) + hideDeleting() + if ((res as any)?.result === 'success') { + hideDeleteConfirm() + onUpdate(true) + } + }, [detail, showDeleting, deleteMCP, hideDeleting, hideDeleteConfirm, onUpdate]) + + useEffect(() => { + if (isTriggerAuthorize) + handleAuthorize() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + if (!detail) + return null + + return ( + <> +
+
+
+ +
+
+
+
{detail.name}
+
+
+ +
copy(detail.server_identifier || '')}>{detail.server_identifier}
+ +
·
+ +
{detail.server_url}
+ +
+
+
+ + + + +
+
+
+ {!isAuthorizing && detail.is_team_authorization && ( + + )} + {!detail.is_team_authorization && !isAuthorizing && ( + + )} + {isAuthorizing && ( + + )} +
+
+
+ {((detail.is_team_authorization && isGettingTools) || isUpdating) && ( + <> +
+
+ {!isUpdating &&
{t('tools.mcp.gettingTools')}
} + {isUpdating &&
{t('tools.mcp.updateTools')}
} +
+
+
+
+ +
+ + )} + {!isUpdating && detail.is_team_authorization && !isGettingTools && !toolList.length && ( +
+
{t('tools.mcp.toolsEmpty')}
+ +
+ )} + {!isUpdating && !isGettingTools && toolList.length > 0 && ( + <> +
+
+ {toolList.length > 1 &&
{t('tools.mcp.toolsNum', { count: toolList.length })}
} + {toolList.length === 1 &&
{t('tools.mcp.onlyTool')}
} +
+
+ +
+
+
+ {toolList.map(tool => ( + + ))} +
+ + )} + + {!isUpdating && !detail.is_team_authorization && ( +
+ {!isAuthorizing &&
{t('tools.mcp.authorizingRequired')}
} + {isAuthorizing &&
{t('tools.mcp.authorizing')}
} +
{t('tools.mcp.authorizeTip')}
+
+ )} +
+ {isShowUpdateModal && ( + + )} + {isShowDeleteConfirm && ( + + {t('tools.mcp.deleteConfirmTitle', { mcp: detail.name })} +
+ } + onCancel={hideDeleteConfirm} + onConfirm={handleDelete} + isLoading={deleting} + isDisabled={deleting} + /> + )} + {isShowUpdateConfirm && ( + + )} + + ) +} + +export default MCPDetailContent diff --git a/web/app/components/tools/mcp/detail/list-loading.tsx b/web/app/components/tools/mcp/detail/list-loading.tsx new file mode 100644 index 0000000000..babf050d8b --- /dev/null +++ b/web/app/components/tools/mcp/detail/list-loading.tsx @@ -0,0 +1,37 @@ +'use client' +import React from 'react' +import cn from '@/utils/classnames' + +const ListLoading = () => { + return ( +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ) +} + +export default ListLoading diff --git a/web/app/components/tools/mcp/detail/operation-dropdown.tsx b/web/app/components/tools/mcp/detail/operation-dropdown.tsx new file mode 100644 index 0000000000..d2cbc8825d --- /dev/null +++ b/web/app/components/tools/mcp/detail/operation-dropdown.tsx @@ -0,0 +1,88 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { + RiDeleteBinLine, + RiEditLine, + RiMoreFill, +} from '@remixicon/react' +import ActionButton from '@/app/components/base/action-button' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' +import cn from '@/utils/classnames' + +type Props = { + inCard?: boolean + onOpenChange?: (open: boolean) => void + onEdit: () => void + onRemove: () => void +} + +const OperationDropdown: FC = ({ + inCard, + onOpenChange, + onEdit, + onRemove, +}) => { + const { t } = useTranslation() + const [open, doSetOpen] = useState(false) + const openRef = useRef(open) + const setOpen = useCallback((v: boolean) => { + doSetOpen(v) + openRef.current = v + onOpenChange?.(v) + }, [doSetOpen]) + + const handleTrigger = useCallback(() => { + setOpen(!openRef.current) + }, [setOpen]) + + return ( + + +
+ + + +
+ + +
+
{ + onEdit() + handleTrigger() + }} + > + +
{t('tools.mcp.operation.edit')}
+
+
{ + onRemove() + handleTrigger() + }} + > + +
{t('tools.mcp.operation.remove')}
+
+
+ + + ) +} +export default React.memo(OperationDropdown) diff --git a/web/app/components/tools/mcp/detail/provider-detail.tsx b/web/app/components/tools/mcp/detail/provider-detail.tsx new file mode 100644 index 0000000000..56f26f8582 --- /dev/null +++ b/web/app/components/tools/mcp/detail/provider-detail.tsx @@ -0,0 +1,56 @@ +'use client' +import React from 'react' +import type { FC } from 'react' +import Drawer from '@/app/components/base/drawer' +import MCPDetailContent from './content' +import type { ToolWithProvider } from '../../../workflow/types' +import cn from '@/utils/classnames' + +type Props = { + detail?: ToolWithProvider + onUpdate: () => void + onHide: () => void + isTriggerAuthorize: boolean + onFirstCreate: () => void +} + +const MCPDetailPanel: FC = ({ + detail, + onUpdate, + onHide, + isTriggerAuthorize, + onFirstCreate, +}) => { + const handleUpdate = (isDelete = false) => { + if (isDelete) + onHide() + onUpdate() + } + + if (!detail) + return null + + return ( + + {detail && ( + + )} + + ) +} + +export default MCPDetailPanel diff --git a/web/app/components/tools/mcp/detail/tool-item.tsx b/web/app/components/tools/mcp/detail/tool-item.tsx new file mode 100644 index 0000000000..dec82edcca --- /dev/null +++ b/web/app/components/tools/mcp/detail/tool-item.tsx @@ -0,0 +1,41 @@ +'use client' +import React from 'react' +import { useContext } from 'use-context-selector' +import type { Tool } from '@/app/components/tools/types' +import I18n from '@/context/i18n' +import { getLanguage } from '@/i18n/language' +import Tooltip from '@/app/components/base/tooltip' +import cn from '@/utils/classnames' + +type Props = { + tool: Tool +} + +const MCPToolItem = ({ + tool, +}: Props) => { + const { locale } = useContext(I18n) + const language = getLanguage(locale) + + return ( + +
{tool.label[language]}
+
{tool.description[language]}
+
+ )} + > +
+
{tool.label[language]}
+
{tool.description[language]}
+
+ + ) +} +export default MCPToolItem diff --git a/web/app/components/tools/mcp/hooks.ts b/web/app/components/tools/mcp/hooks.ts new file mode 100644 index 0000000000..b2b521557f --- /dev/null +++ b/web/app/components/tools/mcp/hooks.ts @@ -0,0 +1,12 @@ +import dayjs from 'dayjs' +import { useCallback } from 'react' +import { useI18N } from '@/context/i18n' + +export const useFormatTimeFromNow = () => { + const { locale } = useI18N() + const formatTimeFromNow = useCallback((time: number) => { + return dayjs(time).locale(locale === 'zh-Hans' ? 'zh-cn' : locale).fromNow() + }, [locale]) + + return { formatTimeFromNow } +} diff --git a/web/app/components/tools/mcp/index.tsx b/web/app/components/tools/mcp/index.tsx new file mode 100644 index 0000000000..5a1e5cf3bf --- /dev/null +++ b/web/app/components/tools/mcp/index.tsx @@ -0,0 +1,98 @@ +'use client' +import { useMemo, useState } from 'react' +import NewMCPCard from './create-card' +import MCPCard from './provider-card' +import MCPDetailPanel from './detail/provider-detail' +import { + useAllToolProviders, +} from '@/service/use-tools' +import type { ToolWithProvider } from '@/app/components/workflow/types' +import cn from '@/utils/classnames' + +type Props = { + searchText: string +} + +function renderDefaultCard() { + const defaultCards = Array.from({ length: 36 }, (_, index) => ( +
= 4 && index < 8 && 'opacity-50', + index >= 8 && index < 12 && 'opacity-40', + index >= 12 && index < 16 && 'opacity-30', + index >= 16 && index < 20 && 'opacity-25', + index >= 20 && index < 24 && 'opacity-20', + )} + >
+ )) + return defaultCards +} + +const MCPList = ({ + searchText, +}: Props) => { + const { data: list = [] as ToolWithProvider[], refetch } = useAllToolProviders() + const [isTriggerAuthorize, setIsTriggerAuthorize] = useState(false) + + const filteredList = useMemo(() => { + return list.filter((collection) => { + if (searchText) + return Object.values(collection.name).some(value => (value as string).toLowerCase().includes(searchText.toLowerCase())) + return collection.type === 'mcp' + }) as ToolWithProvider[] + }, [list, searchText]) + + const [currentProviderID, setCurrentProviderID] = useState() + + const currentProvider = useMemo(() => { + return list.find(provider => provider.id === currentProviderID) + }, [list, currentProviderID]) + + const handleCreate = async (provider: ToolWithProvider) => { + await refetch() // update list + setCurrentProviderID(provider.id) + setIsTriggerAuthorize(true) + } + + const handleUpdate = async (providerID: string) => { + await refetch() // update list + setCurrentProviderID(providerID) + setIsTriggerAuthorize(true) + } + return ( + <> +
+ + {filteredList.map(provider => ( + + ))} + {!list.length && renderDefaultCard()} +
+ {currentProvider && ( + setCurrentProviderID(undefined)} + onUpdate={refetch} + isTriggerAuthorize={isTriggerAuthorize} + onFirstCreate={() => setIsTriggerAuthorize(false)} + /> + )} + + ) +} +export default MCPList diff --git a/web/app/components/tools/mcp/mcp-server-modal.tsx b/web/app/components/tools/mcp/mcp-server-modal.tsx new file mode 100644 index 0000000000..9eb33f21ec --- /dev/null +++ b/web/app/components/tools/mcp/mcp-server-modal.tsx @@ -0,0 +1,134 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import { RiCloseLine } from '@remixicon/react' +import Modal from '@/app/components/base/modal' +import Button from '@/app/components/base/button' +import Textarea from '@/app/components/base/textarea' +import Divider from '@/app/components/base/divider' +import MCPServerParamItem from '@/app/components/tools/mcp/mcp-server-param-item' +import type { + MCPServerDetail, +} from '@/app/components/tools/types' +import { + useCreateMCPServer, + useInvalidateMCPServerDetail, + useUpdateMCPServer, +} from '@/service/use-tools' +import cn from '@/utils/classnames' + +export type ModalProps = { + appID: string + latestParams?: any[] + data?: MCPServerDetail + show: boolean + onHide: () => void +} + +const MCPServerModal = ({ + appID, + latestParams = [], + data, + show, + onHide, +}: ModalProps) => { + const { t } = useTranslation() + const { mutateAsync: createMCPServer, isPending: creating } = useCreateMCPServer() + const { mutateAsync: updateMCPServer, isPending: updating } = useUpdateMCPServer() + const invalidateMCPServerDetail = useInvalidateMCPServerDetail() + + const [description, setDescription] = React.useState(data?.description || '') + const [params, setParams] = React.useState(data?.parameters || {}) + + const handleParamChange = (variable: string, value: string) => { + setParams(prev => ({ + ...prev, + [variable]: value, + })) + } + + const getParamValue = () => { + const res = {} as any + latestParams.map((param) => { + res[param.variable] = params[param.variable] + return param + }) + return res + } + + const submit = async () => { + if (!data) { + await createMCPServer({ + appID, + description, + parameters: getParamValue(), + }) + invalidateMCPServerDetail(appID) + onHide() + } + else { + await updateMCPServer({ + appID, + id: data.id, + description, + parameters: getParamValue(), + }) + invalidateMCPServerDetail(appID) + onHide() + } + } + + return ( + +
+ +
+
+ {!data ? t('tools.mcp.server.modal.addTitle') : t('tools.mcp.server.modal.editTitle')} +
+
+
+
+
{t('tools.mcp.server.modal.description')}
+
*
+
+ +
+ {latestParams.length > 0 && ( +
+
+
{t('tools.mcp.server.modal.parameters')}
+ +
+
{t('tools.mcp.server.modal.parametersTip')}
+
+ {latestParams.map(paramItem => ( + handleParamChange(paramItem.variable, value)} + /> + ))} +
+
+ )} +
+
+ + +
+ + ) +} + +export default MCPServerModal diff --git a/web/app/components/tools/mcp/mcp-server-param-item.tsx b/web/app/components/tools/mcp/mcp-server-param-item.tsx new file mode 100644 index 0000000000..a48d1b92b0 --- /dev/null +++ b/web/app/components/tools/mcp/mcp-server-param-item.tsx @@ -0,0 +1,37 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import Textarea from '@/app/components/base/textarea' + +type Props = { + data?: any + value: string + onChange: (value: string) => void +} + +const MCPServerParamItem = ({ + data, + value, + onChange, +}: Props) => { + const { t } = useTranslation() + + return ( +
+
+
{data.label}
+
·
+
{data.variable}
+
{data.type}
+
+ +
+ ) +} + +export default MCPServerParamItem diff --git a/web/app/components/tools/mcp/mcp-service-card.tsx b/web/app/components/tools/mcp/mcp-service-card.tsx new file mode 100644 index 0000000000..c0c542da26 --- /dev/null +++ b/web/app/components/tools/mcp/mcp-service-card.tsx @@ -0,0 +1,246 @@ +'use client' +import React, { useEffect, useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { RiEditLine, RiLoopLeftLine } from '@remixicon/react' +import { + Mcp, +} from '@/app/components/base/icons/src/vender/other' +import Button from '@/app/components/base/button' +import Tooltip from '@/app/components/base/tooltip' +import Switch from '@/app/components/base/switch' +import Divider from '@/app/components/base/divider' +import CopyFeedback from '@/app/components/base/copy-feedback' +import Confirm from '@/app/components/base/confirm' +import type { AppDetailResponse } from '@/models/app' +import { useAppContext } from '@/context/app-context' +import type { AppSSO } from '@/types/app' +import Indicator from '@/app/components/header/indicator' +import MCPServerModal from '@/app/components/tools/mcp/mcp-server-modal' +import { useAppWorkflow } from '@/service/use-workflow' +import { + useInvalidateMCPServerDetail, + useMCPServerDetail, + useRefreshMCPServerCode, + useUpdateMCPServer, +} from '@/service/use-tools' +import { BlockEnum } from '@/app/components/workflow/types' +import cn from '@/utils/classnames' +import { fetchAppDetail } from '@/service/apps' + +export type IAppCardProps = { + appInfo: AppDetailResponse & Partial +} + +function MCPServiceCard({ + appInfo, +}: IAppCardProps) { + const { t } = useTranslation() + const appId = appInfo.id + const { mutateAsync: updateMCPServer } = useUpdateMCPServer() + const { mutateAsync: refreshMCPServerCode, isPending: genLoading } = useRefreshMCPServerCode() + const invalidateMCPServerDetail = useInvalidateMCPServerDetail() + const { isCurrentWorkspaceManager, isCurrentWorkspaceEditor } = useAppContext() + const [showConfirmDelete, setShowConfirmDelete] = useState(false) + const [showMCPServerModal, setShowMCPServerModal] = useState(false) + + const isAdvancedApp = appInfo?.mode === 'advanced-chat' || appInfo?.mode === 'workflow' + const isBasicApp = !isAdvancedApp + const { data: currentWorkflow } = useAppWorkflow(isAdvancedApp ? appId : '') + const [basicAppConfig, setBasicAppConfig] = useState({}) + const basicAppInputForm = useMemo(() => { + if(!isBasicApp || !basicAppConfig?.user_input_form) + return [] + return basicAppConfig.user_input_form.map((item: any) => { + const type = Object.keys(item)[0] + return { + ...item[type], + type: type || 'text-input', + } + }) + }, [basicAppConfig.user_input_form, isBasicApp]) + useEffect(() => { + if(isBasicApp && appId) { + (async () => { + const res = await fetchAppDetail({ url: '/apps', id: appId }) + setBasicAppConfig(res?.model_config || {}) + })() + } + }, [appId, isBasicApp]) + const { data: detail } = useMCPServerDetail(appId) + const { id, status, server_code } = detail ?? {} + + const appUnpublished = isAdvancedApp ? !currentWorkflow?.graph : !basicAppConfig.updated_at + const serverPublished = !!id + const serverActivated = status === 'active' + const serverURL = serverPublished ? `${appInfo.api_base_url.replace('/v1', '')}/mcp/server/${server_code}/mcp` : '***********' + const toggleDisabled = !isCurrentWorkspaceEditor || appUnpublished + + const [activated, setActivated] = useState(serverActivated) + + const latestParams = useMemo(() => { + if(isAdvancedApp) { + if (!currentWorkflow?.graph) + return [] + const startNode = currentWorkflow?.graph.nodes.find(node => node.data.type === BlockEnum.Start) as any + return startNode?.data.variables as any[] || [] + } + return basicAppInputForm + }, [currentWorkflow, basicAppInputForm, isAdvancedApp]) + + const onGenCode = async () => { + await refreshMCPServerCode(detail?.id || '') + invalidateMCPServerDetail(appId) + } + + const onChangeStatus = async (state: boolean) => { + setActivated(state) + if (state) { + if (!serverPublished) { + setShowMCPServerModal(true) + return + } + + await updateMCPServer({ + appID: appId, + id: id || '', + description: detail?.description || '', + parameters: detail?.parameters || {}, + status: 'active', + }) + invalidateMCPServerDetail(appId) + } + else { + await updateMCPServer({ + appID: appId, + id: id || '', + description: detail?.description || '', + parameters: detail?.parameters || {}, + status: 'inactive', + }) + invalidateMCPServerDetail(appId) + } + } + + const handleServerModalHide = () => { + setShowMCPServerModal(false) + if (!serverActivated) + setActivated(false) + } + + useEffect(() => { + setActivated(serverActivated) + }, [serverActivated]) + + if (!currentWorkflow && isAdvancedApp) + return null + + return ( + <> +
+
+
+
+
+
+ +
+
+
+ {t('tools.mcp.server.title')} +
+
+
+
+ +
+ {serverActivated + ? t('appOverview.overview.status.running') + : t('appOverview.overview.status.disable')} +
+
+ +
+ +
+ +
+
+
+ {t('tools.mcp.server.url')} +
+
+
+
+ {serverURL} +
+
+ {serverPublished && ( + <> + + + {isCurrentWorkspaceManager && ( + +
setShowConfirmDelete(true)} + > + +
+ + )} + + )} +
+
+
+
+ +
+
+
+ {showMCPServerModal && ( + + )} + {/* button copy link/ button regenerate */} + {showConfirmDelete && ( + { + onGenCode() + setShowConfirmDelete(false) + }} + onCancel={() => setShowConfirmDelete(false)} + /> + )} + + ) +} + +export default MCPServiceCard diff --git a/web/app/components/tools/mcp/mock.ts b/web/app/components/tools/mcp/mock.ts new file mode 100644 index 0000000000..f271f67ed3 --- /dev/null +++ b/web/app/components/tools/mcp/mock.ts @@ -0,0 +1,154 @@ +const tools = [ + { + author: 'Novice', + name: 'NOTION_ADD_PAGE_CONTENT', + label: { + en_US: 'NOTION_ADD_PAGE_CONTENT', + zh_Hans: 'NOTION_ADD_PAGE_CONTENT', + pt_BR: 'NOTION_ADD_PAGE_CONTENT', + ja_JP: 'NOTION_ADD_PAGE_CONTENT', + }, + description: { + en_US: 'Adds a single content block to a notion page. multiple calls needed for multiple blocks. note: only supports adding to notion pages. blocks that can contain children: - page (any block type) - toggle (any nested content) - to-do (nested to-dos/blocks) - bulleted list (nested lists/blocks) - numbered list (nested lists/blocks) - callout (child blocks) - quote (nested blocks)', + zh_Hans: 'Adds a single content block to a notion page. multiple calls needed for multiple blocks. note: only supports adding to notion pages. blocks that can contain children: - page (any block type) - toggle (any nested content) - to-do (nested to-dos/blocks) - bulleted list (nested lists/blocks) - numbered list (nested lists/blocks) - callout (child blocks) - quote (nested blocks)', + pt_BR: 'Adds a single content block to a notion page. multiple calls needed for multiple blocks. note: only supports adding to notion pages. blocks that can contain children: - page (any block type) - toggle (any nested content) - to-do (nested to-dos/blocks) - bulleted list (nested lists/blocks) - numbered list (nested lists/blocks) - callout (child blocks) - quote (nested blocks)', + ja_JP: 'Adds a single content block to a notion page. multiple calls needed for multiple blocks. note: only supports adding to notion pages. blocks that can contain children: - page (any block type) - toggle (any nested content) - to-do (nested to-dos/blocks) - bulleted list (nested lists/blocks) - numbered list (nested lists/blocks) - callout (child blocks) - quote (nested blocks)', + }, + parameters: [ + { + name: 'after', + label: { + en_US: 'after', + zh_Hans: 'after', + pt_BR: 'after', + ja_JP: 'after', + }, + placeholder: null, + scope: null, + auto_generate: null, + template: null, + required: false, + default: null, + min: null, + max: null, + precision: null, + options: [], + type: 'string', + human_description: { + en_US: 'The ID of the existing block that the new block should be appended after. If not provided, content will be appended at the end of the page.', + zh_Hans: 'The ID of the existing block that the new block should be appended after. If not provided, content will be appended at the end of the page.', + pt_BR: 'The ID of the existing block that the new block should be appended after. If not provided, content will be appended at the end of the page.', + ja_JP: 'The ID of the existing block that the new block should be appended after. If not provided, content will be appended at the end of the page.', + }, + form: 'llm', + llm_description: 'The ID of the existing block that the new block should be appended after. If not provided, content will be appended at the end of the page.', + }, + { + name: 'content_block', + label: { + en_US: 'content_block', + zh_Hans: 'content_block', + pt_BR: 'content_block', + ja_JP: 'content_block', + }, + placeholder: null, + scope: null, + auto_generate: null, + template: null, + required: false, + default: null, + min: null, + max: null, + precision: null, + options: [], + type: 'string', + human_description: { + en_US: 'Child content to append to a page.', + zh_Hans: 'Child content to append to a page.', + pt_BR: 'Child content to append to a page.', + ja_JP: 'Child content to append to a page.', + }, + form: 'llm', + llm_description: 'Child content to append to a page.', + }, + { + name: 'parent_block_id', + label: { + en_US: 'parent_block_id', + zh_Hans: 'parent_block_id', + pt_BR: 'parent_block_id', + ja_JP: 'parent_block_id', + }, + placeholder: null, + scope: null, + auto_generate: null, + template: null, + required: false, + default: null, + min: null, + max: null, + precision: null, + options: [], + type: 'string', + human_description: { + en_US: 'The ID of the page which the children will be added.', + zh_Hans: 'The ID of the page which the children will be added.', + pt_BR: 'The ID of the page which the children will be added.', + ja_JP: 'The ID of the page which the children will be added.', + }, + form: 'llm', + llm_description: 'The ID of the page which the children will be added.', + }, + ], + labels: [], + output_schema: null, + }, +] + +export const listData = [ + { + id: 'fdjklajfkljadslf111', + author: 'KVOJJJin', + name: 'GOGOGO', + icon: 'https://cloud.dify.dev/console/api/workspaces/694cc430-fa36-4458-86a0-4a98c09c4684/model-providers/langgenius/openai/openai/icon_small/en_US', + server_url: 'https://mcp.composio.dev/notion/****/abc', + type: 'mcp', + is_team_authorization: true, + tools, + update_elapsed_time: 1744793369, + label: { + en_US: 'GOGOGO', + zh_Hans: 'GOGOGO', + }, + }, + { + id: 'fdjklajfkljadslf222', + author: 'KVOJJJin', + name: 'GOGOGO2', + icon: 'https://cloud.dify.dev/console/api/workspaces/694cc430-fa36-4458-86a0-4a98c09c4684/model-providers/langgenius/openai/openai/icon_small/en_US', + server_url: 'https://mcp.composio.dev/notion/****/abc', + type: 'mcp', + is_team_authorization: false, + tools: [], + update_elapsed_time: 1744793369, + label: { + en_US: 'GOGOGO2', + zh_Hans: 'GOGOGO2', + }, + }, + { + id: 'fdjklajfkljadslf333', + author: 'KVOJJJin', + name: 'GOGOGO3', + icon: 'https://cloud.dify.dev/console/api/workspaces/694cc430-fa36-4458-86a0-4a98c09c4684/model-providers/langgenius/openai/openai/icon_small/en_US', + server_url: 'https://mcp.composio.dev/notion/****/abc', + type: 'mcp', + is_team_authorization: true, + tools, + update_elapsed_time: 1744793369, + label: { + en_US: 'GOGOGO3', + zh_Hans: 'GOGOGO3', + }, + }, +] diff --git a/web/app/components/tools/mcp/modal.tsx b/web/app/components/tools/mcp/modal.tsx new file mode 100644 index 0000000000..0e57cb149b --- /dev/null +++ b/web/app/components/tools/mcp/modal.tsx @@ -0,0 +1,221 @@ +'use client' +import React, { useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { getDomain } from 'tldts' +import { RiCloseLine, RiEditLine } from '@remixicon/react' +import AppIconPicker from '@/app/components/base/app-icon-picker' +import type { AppIconSelection } from '@/app/components/base/app-icon-picker' +import AppIcon from '@/app/components/base/app-icon' +import Modal from '@/app/components/base/modal' +import Button from '@/app/components/base/button' +import Input from '@/app/components/base/input' +import type { AppIconType } from '@/types/app' +import type { ToolWithProvider } from '@/app/components/workflow/types' +import { noop } from 'lodash-es' +import Toast from '@/app/components/base/toast' +import { uploadRemoteFileInfo } from '@/service/common' +import cn from '@/utils/classnames' +import { useHover } from 'ahooks' + +export type DuplicateAppModalProps = { + data?: ToolWithProvider + show: boolean + onConfirm: (info: { + name: string + server_url: string + icon_type: AppIconType + icon: string + icon_background?: string | null + server_identifier: string + }) => void + onHide: () => void +} + +const DEFAULT_ICON = { type: 'emoji', icon: '🧿', background: '#EFF1F5' } +const extractFileId = (url: string) => { + const match = url.match(/files\/(.+?)\/file-preview/) + return match ? match[1] : null +} +const getIcon = (data?: ToolWithProvider) => { + if (!data) + return DEFAULT_ICON as AppIconSelection + if (typeof data.icon === 'string') + return { type: 'image', url: data.icon, fileId: extractFileId(data.icon) } as AppIconSelection + return { + ...data.icon, + icon: data.icon.content, + type: 'emoji', + } as unknown as AppIconSelection +} + +const MCPModal = ({ + data, + show, + onConfirm, + onHide, +}: DuplicateAppModalProps) => { + const { t } = useTranslation() + const isCreate = !data + + const originalServerUrl = data?.server_url + const originalServerID = data?.server_identifier + const [url, setUrl] = React.useState(data?.server_url || '') + const [name, setName] = React.useState(data?.name || '') + const [appIcon, setAppIcon] = useState(getIcon(data)) + const [showAppIconPicker, setShowAppIconPicker] = useState(false) + const [serverIdentifier, setServerIdentifier] = React.useState(data?.server_identifier || '') + const [isFetchingIcon, setIsFetchingIcon] = useState(false) + const appIconRef = useRef(null) + const isHovering = useHover(appIconRef) + + const isValidUrl = (string: string) => { + try { + const urlPattern = /^(https?:\/\/)((([a-z\d]([a-z\d-]*[a-z\d])*)\.)+[a-z]{2,}|((\d{1,3}\.){3}\d{1,3}))(\:\d+)?(\/[-a-z\d%_.~+]*)*(\?[;&a-z\d%_.~+=-]*)?/i + return urlPattern.test(string) + } + catch (e) { + return false + } + } + + const isValidServerID = (str: string) => { + return /^[a-z0-9_-]{1,24}$/.test(str) + } + + const handleBlur = async (url: string) => { + if (data) + return + if (!isValidUrl(url)) + return + const domain = getDomain(url) + const remoteIcon = `https://www.google.com/s2/favicons?domain=${domain}&sz=128` + setIsFetchingIcon(true) + try { + const res = await uploadRemoteFileInfo(remoteIcon, undefined, true) + setAppIcon({ type: 'image', url: res.url, fileId: extractFileId(res.url) || '' }) + } + catch (e) { + console.error('Failed to fetch remote icon:', e) + Toast.notify({ type: 'warning', message: 'Failed to fetch remote icon' }) + } + finally { + setIsFetchingIcon(false) + } + } + + const submit = async () => { + if (!isValidUrl(url)) { + Toast.notify({ type: 'error', message: 'invalid server url' }) + return + } + if (!isValidServerID(serverIdentifier.trim())) { + Toast.notify({ type: 'error', message: 'invalid server identifier' }) + return + } + await onConfirm({ + server_url: originalServerUrl === url ? '[__HIDDEN__]' : url.trim(), + name, + icon_type: appIcon.type, + icon: appIcon.type === 'emoji' ? appIcon.icon : appIcon.fileId, + icon_background: appIcon.type === 'emoji' ? appIcon.background : undefined, + server_identifier: serverIdentifier.trim(), + }) + if(isCreate) + onHide() + } + + return ( + <> + +
+ +
+
{!isCreate ? t('tools.mcp.modal.editTitle') : t('tools.mcp.modal.title')}
+
+
+
+ {t('tools.mcp.modal.serverUrl')} +
+ setUrl(e.target.value)} + onBlur={e => handleBlur(e.target.value.trim())} + placeholder={t('tools.mcp.modal.serverUrlPlaceholder')} + /> + {originalServerUrl && originalServerUrl !== url && ( +
+ {t('tools.mcp.modal.serverUrlWarning')} +
+ )} +
+
+
+
+ {t('tools.mcp.modal.name')} +
+ setName(e.target.value)} + placeholder={t('tools.mcp.modal.namePlaceholder')} + /> +
+
+ + +
) : null + } + onClick={() => { setShowAppIconPicker(true) }} + /> +
+
+
+
+ {t('tools.mcp.modal.serverIdentifier')} +
+
{t('tools.mcp.modal.serverIdentifierTip')}
+ setServerIdentifier(e.target.value)} + placeholder={t('tools.mcp.modal.serverIdentifierPlaceholder')} + /> + {originalServerID && originalServerID !== serverIdentifier && ( +
+ {t('tools.mcp.modal.serverIdentifierWarning')} +
+ )} +
+
+
+ + +
+ + {showAppIconPicker && { + setAppIcon(payload) + setShowAppIconPicker(false) + }} + onClose={() => { + setAppIcon(getIcon(data)) + setShowAppIconPicker(false) + }} + />} + + + ) +} + +export default MCPModal diff --git a/web/app/components/tools/mcp/provider-card.tsx b/web/app/components/tools/mcp/provider-card.tsx new file mode 100644 index 0000000000..677e25c533 --- /dev/null +++ b/web/app/components/tools/mcp/provider-card.tsx @@ -0,0 +1,152 @@ +'use client' +import { useCallback, useState } from 'react' +import { useBoolean } from 'ahooks' +import { useTranslation } from 'react-i18next' +import { useAppContext } from '@/context/app-context' +import { RiHammerFill } from '@remixicon/react' +import Indicator from '@/app/components/header/indicator' +import Icon from '@/app/components/plugins/card/base/card-icon' +import { useFormatTimeFromNow } from './hooks' +import type { ToolWithProvider } from '../../workflow/types' +import Confirm from '@/app/components/base/confirm' +import MCPModal from './modal' +import OperationDropdown from './detail/operation-dropdown' +import { useDeleteMCP, useUpdateMCP } from '@/service/use-tools' +import cn from '@/utils/classnames' + +type Props = { + currentProvider?: ToolWithProvider + data: ToolWithProvider + handleSelect: (providerID: string) => void + onUpdate: (providerID: string) => void + onDeleted: () => void +} + +const MCPCard = ({ + currentProvider, + data, + onUpdate, + handleSelect, + onDeleted, +}: Props) => { + const { t } = useTranslation() + const { formatTimeFromNow } = useFormatTimeFromNow() + const { isCurrentWorkspaceManager } = useAppContext() + + const { mutateAsync: updateMCP } = useUpdateMCP({}) + const { mutateAsync: deleteMCP } = useDeleteMCP({}) + + const [isOperationShow, setIsOperationShow] = useState(false) + + const [isShowUpdateModal, { + setTrue: showUpdateModal, + setFalse: hideUpdateModal, + }] = useBoolean(false) + + const [isShowDeleteConfirm, { + setTrue: showDeleteConfirm, + setFalse: hideDeleteConfirm, + }] = useBoolean(false) + + const [deleting, { + setTrue: showDeleting, + setFalse: hideDeleting, + }] = useBoolean(false) + + const handleUpdate = useCallback(async (form: any) => { + const res = await updateMCP({ + ...form, + provider_id: data.id, + }) + if ((res as any)?.result === 'success') { + hideUpdateModal() + onUpdate(data.id) + } + }, [data, updateMCP, hideUpdateModal, onUpdate]) + + const handleDelete = useCallback(async () => { + showDeleting() + const res = await deleteMCP(data.id) + hideDeleting() + if ((res as any)?.result === 'success') { + hideDeleteConfirm() + onDeleted() + } + }, [showDeleting, deleteMCP, data.id, hideDeleting, hideDeleteConfirm, onDeleted]) + + return ( +
handleSelect(data.id)} + className={cn( + 'group relative flex cursor-pointer flex-col rounded-xl border-[1.5px] border-transparent bg-components-card-bg shadow-xs hover:bg-components-card-bg-alt hover:shadow-md', + currentProvider?.id === data.id && 'border-components-option-card-option-selected-border bg-components-card-bg-alt', + )} + > +
+
+ +
+
+
{data.name}
+
{data.server_identifier}
+
+
+
+
+
+ + {data.tools.length > 0 && ( +
{t('tools.mcp.toolsCount', { count: data.tools.length })}
+ )} + {!data.tools.length && ( +
{t('tools.mcp.noTools')}
+ )} +
+
/
+
{`${t('tools.mcp.updateTime')} ${formatTimeFromNow(data.updated_at! * 1000)}`}
+
+ {data.is_team_authorization && data.tools.length > 0 && } + {(!data.is_team_authorization || !data.tools.length) && ( +
+ {t('tools.mcp.noConfigured')} + +
+ )} +
+ {isCurrentWorkspaceManager && ( + + )} + {isShowUpdateModal && ( + + )} + {isShowDeleteConfirm && ( + + {t('tools.mcp.deleteConfirmTitle', { mcp: data.name })} +
+ } + onCancel={hideDeleteConfirm} + onConfirm={handleDelete} + isLoading={deleting} + isDisabled={deleting} + /> + )} +
+ ) +} +export default MCPCard diff --git a/web/app/components/tools/provider-list.tsx b/web/app/components/tools/provider-list.tsx index b0b4f8a8bc..ecfa5f6ea2 100644 --- a/web/app/components/tools/provider-list.tsx +++ b/web/app/components/tools/provider-list.tsx @@ -15,11 +15,29 @@ import WorkflowToolEmpty from '@/app/components/tools/add-tool-modal/empty' import Card from '@/app/components/plugins/card' import CardMoreInfo from '@/app/components/plugins/card/card-more-info' import PluginDetailPanel from '@/app/components/plugins/plugin-detail-panel' +import MCPList from './mcp' import { useAllToolProviders } from '@/service/use-tools' import { useInstalledPluginList, useInvalidateInstalledPluginList } from '@/service/use-plugins' import { useGlobalPublicStore } from '@/context/global-public-context' +import { ToolTypeEnum } from '../workflow/block-selector/types' +const getToolType = (type: string) => { + switch (type) { + case 'builtin': + return ToolTypeEnum.BuiltIn + case 'api': + return ToolTypeEnum.Custom + case 'workflow': + return ToolTypeEnum.Workflow + case 'mcp': + return ToolTypeEnum.MCP + default: + return ToolTypeEnum.BuiltIn + } +} const ProviderList = () => { + // const searchParams = useSearchParams() + // searchParams.get('category') === 'workflow' const { t } = useTranslation() const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) const containerRef = useRef(null) @@ -31,6 +49,7 @@ const ProviderList = () => { { value: 'builtin', text: t('tools.type.builtIn') }, { value: 'api', text: t('tools.type.custom') }, { value: 'workflow', text: t('tools.type.workflow') }, + { value: 'mcp', text: 'MCP' }, ] const [tagFilterValue, setTagFilterValue] = useState([]) const handleTagsChange = (value: string[]) => { @@ -85,7 +104,9 @@ const ProviderList = () => { options={options} />
- + {activeTab !== 'mcp' && ( + + )} { />
- {(filteredCollectionList.length > 0 || activeTab !== 'builtin') && ( + {activeTab !== 'mcp' && (
{ />
))} - {!filteredCollectionList.length && activeTab === 'workflow' &&
} + {!filteredCollectionList.length && activeTab === 'workflow' &&
}
)} {!filteredCollectionList.length && activeTab === 'builtin' && ( )} - { - enable_marketplace && activeTab === 'builtin' && ( - { - containerRef.current?.scrollTo({ top: containerRef.current.scrollHeight, behavior: 'smooth' }) - }} - searchPluginText={keywords} - filterPluginTags={tagFilterValue} - /> - ) - } -
-
+ {enable_marketplace && activeTab === 'builtin' && ( + { + containerRef.current?.scrollTo({ top: containerRef.current.scrollHeight, behavior: 'smooth' }) + }} + searchPluginText={keywords} + filterPluginTags={tagFilterValue} + /> + )} + {activeTab === 'mcp' && ( + + )} +
+
{currentProvider && !currentProvider.plugin_id && ( { return ( <> {isCurrentWorkspaceManager && ( -
-
setIsShowEditCustomCollectionModal(true)}> +
+
setIsShowEditCustomCollectionModal(true)}>
-
- +
+
-
{t('tools.createCustomTool')}
+
{t('tools.createCustomTool')}
- diff --git a/web/app/components/tools/provider/tool-item.tsx b/web/app/components/tools/provider/tool-item.tsx index 161b62963b..d79d20cb9c 100644 --- a/web/app/components/tools/provider/tool-item.tsx +++ b/web/app/components/tools/provider/tool-item.tsx @@ -29,7 +29,7 @@ const ToolItem = ({ return ( <>
!disabled && setShowDetail(true)} >
{tool.label[language]}
diff --git a/web/app/components/tools/types.ts b/web/app/components/tools/types.ts index 32c468cde8..b83919ad18 100644 --- a/web/app/components/tools/types.ts +++ b/web/app/components/tools/types.ts @@ -7,7 +7,8 @@ export enum LOC { export enum AuthType { none = 'none', - apiKey = 'api_key', + apiKeyHeader = 'api_key_header', + apiKeyQuery = 'api_key_query', } export enum AuthHeaderPrefix { @@ -21,6 +22,7 @@ export type Credential = { api_key_header?: string api_key_value?: string api_key_header_prefix?: AuthHeaderPrefix + api_key_query_param?: string } export enum CollectionType { @@ -29,6 +31,7 @@ export enum CollectionType { custom = 'api', model = 'model', workflow = 'workflow', + mcp = 'mcp', } export type Emoji = { @@ -50,6 +53,10 @@ export type Collection = { labels: string[] plugin_id?: string letter?: string + // MCP Server + server_url?: string + updated_at?: number + server_identifier?: string } export type ToolParameter = { @@ -168,3 +175,11 @@ export type WorkflowToolProviderResponse = { } privacy_policy: string } + +export type MCPServerDetail = { + id: string + server_code: string + description: string + status: string + parameters?: Record +} diff --git a/web/app/components/tools/utils/to-form-schema.ts b/web/app/components/tools/utils/to-form-schema.ts index 179f59021e..ee7f3379ad 100644 --- a/web/app/components/tools/utils/to-form-schema.ts +++ b/web/app/components/tools/utils/to-form-schema.ts @@ -1,4 +1,7 @@ import type { ToolCredential, ToolParameter } from '../types' +import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' + export const toType = (type: string) => { switch (type) { case 'string': @@ -54,7 +57,7 @@ export const toolCredentialToFormSchemas = (parameters: ToolCredential[]) => { return formSchemas } -export const addDefaultValue = (value: Record, formSchemas: { variable: string; default?: any }[]) => { +export const addDefaultValue = (value: Record, formSchemas: { variable: string; type: string; default?: any }[]) => { const newValues = { ...value } formSchemas.forEach((formSchema) => { const itemValue = value[formSchema.variable] @@ -64,14 +67,47 @@ export const addDefaultValue = (value: Record, formSchemas: { varia return newValues } -export const generateFormValue = (value: Record, formSchemas: { variable: string; default?: any }[], isReasoning = false) => { +const correctInitialData = (type: string, target: any, defaultValue: any) => { + if (type === 'text-input' || type === 'secret-input') + target.type = 'mixed' + + if (type === 'boolean') { + if (typeof defaultValue === 'string') + target.value = defaultValue === 'true' || defaultValue === '1' + + if (typeof defaultValue === 'boolean') + target.value = defaultValue + + if (typeof defaultValue === 'number') + target.value = defaultValue === 1 + } + + if (type === 'number-input') { + if (typeof defaultValue === 'string' && defaultValue !== '') + target.value = Number.parseFloat(defaultValue) + } + + if (type === 'app-selector' || type === 'model-selector') + target.value = defaultValue + + return target +} + +export const generateFormValue = (value: Record, formSchemas: { variable: string; default?: any; type: string }[], isReasoning = false) => { const newValues = {} as any formSchemas.forEach((formSchema) => { const itemValue = value[formSchema.variable] if ((formSchema.default !== undefined) && (value === undefined || itemValue === null || itemValue === '' || itemValue === undefined)) { + const value = formSchema.default newValues[formSchema.variable] = { - ...(isReasoning ? { value: null, auto: 1 } : { value: formSchema.default }), + value: { + type: 'constant', + value: formSchema.default, + }, + ...(isReasoning ? { auto: 1, value: null } : {}), } + if (!isReasoning) + newValues[formSchema.variable].value = correctInitialData(formSchema.type, newValues[formSchema.variable].value, value) } }) return newValues @@ -80,7 +116,9 @@ export const generateFormValue = (value: Record, formSchemas: { var export const getPlainValue = (value: Record) => { const plainValue = { ...value } Object.keys(plainValue).forEach((key) => { - plainValue[key] = value[key].value + plainValue[key] = { + ...value[key].value, + } }) return plainValue } @@ -94,3 +132,65 @@ export const getStructureValue = (value: Record) => { }) return newValue } + +export const getConfiguredValue = (value: Record, formSchemas: { variable: string; type: string; default?: any }[]) => { + const newValues = { ...value } + formSchemas.forEach((formSchema) => { + const itemValue = value[formSchema.variable] + if ((formSchema.default !== undefined) && (value === undefined || itemValue === null || itemValue === '' || itemValue === undefined)) { + const value = formSchema.default + newValues[formSchema.variable] = { + type: 'constant', + value: formSchema.default, + } + newValues[formSchema.variable] = correctInitialData(formSchema.type, newValues[formSchema.variable], value) + } + }) + return newValues +} + +const getVarKindType = (type: FormTypeEnum) => { + if (type === FormTypeEnum.file || type === FormTypeEnum.files) + return VarKindType.variable + if (type === FormTypeEnum.select || type === FormTypeEnum.boolean || type === FormTypeEnum.textNumber) + return VarKindType.constant + if (type === FormTypeEnum.textInput || type === FormTypeEnum.secretInput) + return VarKindType.mixed + } + +export const generateAgentToolValue = (value: Record, formSchemas: { variable: string; default?: any; type: string }[], isReasoning = false) => { + const newValues = {} as any + if (!isReasoning) { + formSchemas.forEach((formSchema) => { + const itemValue = value[formSchema.variable] + newValues[formSchema.variable] = { + value: { + type: 'constant', + value: itemValue.value, + }, + } + newValues[formSchema.variable].value = correctInitialData(formSchema.type, newValues[formSchema.variable].value, itemValue.value) + }) + } + else { + formSchemas.forEach((formSchema) => { + const itemValue = value[formSchema.variable] + if (itemValue.auto === 1) { + newValues[formSchema.variable] = { + auto: 1, + value: null, + } + } + else { + newValues[formSchema.variable] = { + auto: 0, + value: itemValue.value || { + type: getVarKindType(formSchema.type as FormTypeEnum), + value: null, + }, + } + } + }) + } + return newValues +} diff --git a/web/app/components/workflow-app/components/workflow-header/features-trigger.tsx b/web/app/components/workflow-app/components/workflow-header/features-trigger.tsx index 83f354d7d8..fd48935147 100644 --- a/web/app/components/workflow-app/components/workflow-header/features-trigger.tsx +++ b/web/app/components/workflow-app/components/workflow-header/features-trigger.tsx @@ -23,7 +23,7 @@ import { InputVarType, } from '@/app/components/workflow/types' import { useToastContext } from '@/app/components/base/toast' -import { usePublishWorkflow, useResetWorkflowVersionHistory } from '@/service/use-workflow' +import { useInvalidateAppWorkflow, usePublishWorkflow, useResetWorkflowVersionHistory } from '@/service/use-workflow' import type { PublishWorkflowParams } from '@/types/workflow' import { fetchAppDetail } from '@/service/apps' import { useStore as useAppStore } from '@/app/components/app/store' @@ -89,6 +89,7 @@ const FeaturesTrigger = () => { } }, [appID, setAppDetail]) const { mutateAsync: publishWorkflow } = usePublishWorkflow(appID!) + const updatePublishedWorkflow = useInvalidateAppWorkflow() const onPublish = useCallback(async (params?: PublishWorkflowParams) => { if (await handleCheckBeforePublish()) { const res = await publishWorkflow({ @@ -98,6 +99,7 @@ const FeaturesTrigger = () => { if (res) { notify({ type: 'success', message: t('common.api.actionSuccess') }) + updatePublishedWorkflow(appID!) updateAppDetail() workflowStore.getState().setPublishedAt(res.created_at) resetWorkflowVersionHistory() @@ -106,7 +108,7 @@ const FeaturesTrigger = () => { else { throw new Error('Checklist failed') } - }, [handleCheckBeforePublish, notify, t, workflowStore, publishWorkflow, resetWorkflowVersionHistory, updateAppDetail]) + }, [handleCheckBeforePublish, publishWorkflow, notify, t, updatePublishedWorkflow, appID, updateAppDetail, workflowStore, resetWorkflowVersionHistory]) const onPublisherToggle = useCallback((state: boolean) => { if (state) diff --git a/web/app/components/workflow-app/components/workflow-main.tsx b/web/app/components/workflow-app/components/workflow-main.tsx index f0b9bab2cf..d425e6f595 100644 --- a/web/app/components/workflow-app/components/workflow-main.tsx +++ b/web/app/components/workflow-app/components/workflow-main.tsx @@ -15,7 +15,7 @@ import { useWorkflowRun, useWorkflowStartRun, } from '../hooks' -import { useWorkflowStore } from '@/app/components/workflow/store' +import { useStore, useWorkflowStore } from '@/app/components/workflow/store' type WorkflowMainProps = Pick const WorkflowMain = ({ @@ -64,7 +64,11 @@ const WorkflowMain = ({ handleWorkflowStartRunInChatflow, handleWorkflowStartRunInWorkflow, } = useWorkflowStartRun() - const { fetchInspectVars } = useSetWorkflowVarsWithValue() + const appId = useStore(s => s.appId) + const { fetchInspectVars } = useSetWorkflowVarsWithValue({ + flowId: appId, + ...useConfigsMap(), + }) const { hasNodeInspectVars, hasSetInspectVar, diff --git a/web/app/components/workflow-app/hooks/index.ts b/web/app/components/workflow-app/hooks/index.ts index 1ee7c030b9..9e4f94965b 100644 --- a/web/app/components/workflow-app/hooks/index.ts +++ b/web/app/components/workflow-app/hooks/index.ts @@ -5,6 +5,6 @@ export * from './use-workflow-run' export * from './use-workflow-start-run' export * from './use-is-chat-mode' export * from './use-workflow-refresh-draft' -export * from './use-fetch-workflow-inspect-vars' +export * from '../../workflow/hooks/use-fetch-workflow-inspect-vars' export * from './use-inspect-vars-crud' export * from './use-configs-map' diff --git a/web/app/components/workflow-app/hooks/use-inspect-vars-crud.ts b/web/app/components/workflow-app/hooks/use-inspect-vars-crud.ts index ce052b7ed4..109ee85f96 100644 --- a/web/app/components/workflow-app/hooks/use-inspect-vars-crud.ts +++ b/web/app/components/workflow-app/hooks/use-inspect-vars-crud.ts @@ -1,234 +1,16 @@ -import { fetchNodeInspectVars } from '@/service/workflow' -import { useStore, useWorkflowStore } from '@/app/components/workflow/store' -import type { ValueSelector } from '@/app/components/workflow/types' -import type { VarInInspect } from '@/types/workflow' -import { VarInInspectType } from '@/types/workflow' -import { - useDeleteAllInspectorVars, - useDeleteInspectVar, - useDeleteNodeInspectorVars, - useEditInspectorVar, - useInvalidateConversationVarValues, - useInvalidateSysVarValues, - useResetConversationVar, - useResetToLastRunValue, -} from '@/service/use-workflow' -import { useCallback } from 'react' -import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' -import produce from 'immer' -import type { Node } from '@/app/components/workflow/types' -import { useNodesInteractionsWithoutSync } from '@/app/components/workflow/hooks/use-nodes-interactions-without-sync' -import { useEdgesInteractionsWithoutSync } from '@/app/components/workflow/hooks/use-edges-interactions-without-sync' +import { useStore } from '@/app/components/workflow/store' +import { useInspectVarsCrudCommon } from '../../workflow/hooks/use-inspect-vars-crud-common' import { useConfigsMap } from './use-configs-map' export const useInspectVarsCrud = () => { - const workflowStore = useWorkflowStore() const appId = useStore(s => s.appId) - const { conversationVarsUrl, systemVarsUrl } = useConfigsMap() - const invalidateConversationVarValues = useInvalidateConversationVarValues(conversationVarsUrl) - const { mutateAsync: doResetConversationVar } = useResetConversationVar(appId) - const { mutateAsync: doResetToLastRunValue } = useResetToLastRunValue(appId) - const invalidateSysVarValues = useInvalidateSysVarValues(systemVarsUrl) - - const { mutateAsync: doDeleteAllInspectorVars } = useDeleteAllInspectorVars(appId) - const { mutate: doDeleteNodeInspectorVars } = useDeleteNodeInspectorVars(appId) - const { mutate: doDeleteInspectVar } = useDeleteInspectVar(appId) - - const { mutateAsync: doEditInspectorVar } = useEditInspectorVar(appId) - const { handleCancelNodeSuccessStatus } = useNodesInteractionsWithoutSync() - const { handleEdgeCancelRunningStatus } = useEdgesInteractionsWithoutSync() - const getNodeInspectVars = useCallback((nodeId: string) => { - const { nodesWithInspectVars } = workflowStore.getState() - const node = nodesWithInspectVars.find(node => node.nodeId === nodeId) - return node - }, [workflowStore]) - - const getVarId = useCallback((nodeId: string, varName: string) => { - const node = getNodeInspectVars(nodeId) - if (!node) - return undefined - const varId = node.vars.find((varItem) => { - return varItem.selector[1] === varName - })?.id - return varId - }, [getNodeInspectVars]) - - const getInspectVar = useCallback((nodeId: string, name: string): VarInInspect | undefined => { - const node = getNodeInspectVars(nodeId) - if (!node) - return undefined - - const variable = node.vars.find((varItem) => { - return varItem.name === name - }) - return variable - }, [getNodeInspectVars]) - - const hasSetInspectVar = useCallback((nodeId: string, name: string, sysVars: VarInInspect[], conversationVars: VarInInspect[]) => { - const isEnv = isENV([nodeId]) - if (isEnv) // always have value - return true - const isSys = isSystemVar([nodeId]) - if (isSys) - return sysVars.some(varItem => varItem.selector?.[1]?.replace('sys.', '') === name) - const isChatVar = isConversationVar([nodeId]) - if (isChatVar) - return conversationVars.some(varItem => varItem.selector?.[1] === name) - return getInspectVar(nodeId, name) !== undefined - }, [getInspectVar]) - - const hasNodeInspectVars = useCallback((nodeId: string) => { - return !!getNodeInspectVars(nodeId) - }, [getNodeInspectVars]) - - const fetchInspectVarValue = useCallback(async (selector: ValueSelector) => { - const { - appId, - setNodeInspectVars, - } = workflowStore.getState() - const nodeId = selector[0] - const isSystemVar = nodeId === 'sys' - const isConversationVar = nodeId === 'conversation' - if (isSystemVar) { - invalidateSysVarValues() - return - } - if (isConversationVar) { - invalidateConversationVarValues() - return - } - const vars = await fetchNodeInspectVars(appId, nodeId) - setNodeInspectVars(nodeId, vars) - }, [workflowStore, invalidateSysVarValues, invalidateConversationVarValues]) - - // after last run would call this - const appendNodeInspectVars = useCallback((nodeId: string, payload: VarInInspect[], allNodes: Node[]) => { - const { - nodesWithInspectVars, - setNodesWithInspectVars, - } = workflowStore.getState() - const nodes = produce(nodesWithInspectVars, (draft) => { - const nodeInfo = allNodes.find(node => node.id === nodeId) - if (nodeInfo) { - const index = draft.findIndex(node => node.nodeId === nodeId) - if (index === -1) { - draft.unshift({ - nodeId, - nodeType: nodeInfo.data.type, - title: nodeInfo.data.title, - vars: payload, - nodePayload: nodeInfo.data, - }) - } - else { - draft[index].vars = payload - // put the node to the topAdd commentMore actions - draft.unshift(draft.splice(index, 1)[0]) - } - } - }) - setNodesWithInspectVars(nodes) - handleCancelNodeSuccessStatus(nodeId) - }, [workflowStore, handleCancelNodeSuccessStatus]) - - const hasNodeInspectVar = useCallback((nodeId: string, varId: string) => { - const { nodesWithInspectVars } = workflowStore.getState() - const targetNode = nodesWithInspectVars.find(item => item.nodeId === nodeId) - if(!targetNode || !targetNode.vars) - return false - return targetNode.vars.some(item => item.id === varId) - }, [workflowStore]) - - const deleteInspectVar = useCallback(async (nodeId: string, varId: string) => { - const { deleteInspectVar } = workflowStore.getState() - if(hasNodeInspectVar(nodeId, varId)) { - await doDeleteInspectVar(varId) - deleteInspectVar(nodeId, varId) - } - }, [doDeleteInspectVar, workflowStore, hasNodeInspectVar]) - - const resetConversationVar = useCallback(async (varId: string) => { - await doResetConversationVar(varId) - invalidateConversationVarValues() - }, [doResetConversationVar, invalidateConversationVarValues]) - - const deleteNodeInspectorVars = useCallback(async (nodeId: string) => { - const { deleteNodeInspectVars } = workflowStore.getState() - if (hasNodeInspectVars(nodeId)) { - await doDeleteNodeInspectorVars(nodeId) - deleteNodeInspectVars(nodeId) - } - }, [doDeleteNodeInspectorVars, workflowStore, hasNodeInspectVars]) - - const deleteAllInspectorVars = useCallback(async () => { - const { deleteAllInspectVars } = workflowStore.getState() - await doDeleteAllInspectorVars() - await invalidateConversationVarValues() - await invalidateSysVarValues() - deleteAllInspectVars() - handleEdgeCancelRunningStatus() - }, [doDeleteAllInspectorVars, invalidateConversationVarValues, invalidateSysVarValues, workflowStore, handleEdgeCancelRunningStatus]) - - const editInspectVarValue = useCallback(async (nodeId: string, varId: string, value: any) => { - const { setInspectVarValue } = workflowStore.getState() - await doEditInspectorVar({ - varId, - value, - }) - setInspectVarValue(nodeId, varId, value) - if (nodeId === VarInInspectType.conversation) - invalidateConversationVarValues() - if (nodeId === VarInInspectType.system) - invalidateSysVarValues() - }, [doEditInspectorVar, invalidateConversationVarValues, invalidateSysVarValues, workflowStore]) - - const renameInspectVarName = useCallback(async (nodeId: string, oldName: string, newName: string) => { - const { renameInspectVarName } = workflowStore.getState() - const varId = getVarId(nodeId, oldName) - if (!varId) - return - - const newSelector = [nodeId, newName] - await doEditInspectorVar({ - varId, - name: newName, - }) - renameInspectVarName(nodeId, varId, newSelector) - }, [doEditInspectorVar, getVarId, workflowStore]) - - const isInspectVarEdited = useCallback((nodeId: string, name: string) => { - const inspectVar = getInspectVar(nodeId, name) - if (!inspectVar) - return false - - return inspectVar.edited - }, [getInspectVar]) - - const resetToLastRunVar = useCallback(async (nodeId: string, varId: string) => { - const { resetToLastRunVar } = workflowStore.getState() - const isSysVar = nodeId === 'sys' - const data = await doResetToLastRunValue(varId) - - if(isSysVar) - invalidateSysVarValues() - else - resetToLastRunVar(nodeId, varId, data.value) - }, [doResetToLastRunValue, invalidateSysVarValues, workflowStore]) + const configsMap = useConfigsMap() + const apis = useInspectVarsCrudCommon({ + flowId: appId, + ...configsMap, + }) return { - hasNodeInspectVars, - hasSetInspectVar, - fetchInspectVarValue, - editInspectVarValue, - renameInspectVarName, - appendNodeInspectVars, - deleteInspectVar, - deleteNodeInspectorVars, - deleteAllInspectorVars, - isInspectVarEdited, - resetToLastRunVar, - invalidateSysVarValues, - resetConversationVar, - invalidateConversationVarValues, + ...apis, } } diff --git a/web/app/components/workflow-app/hooks/use-workflow-run.ts b/web/app/components/workflow-app/hooks/use-workflow-run.ts index 4c34d2ffb1..c303211715 100644 --- a/web/app/components/workflow-app/hooks/use-workflow-run.ts +++ b/web/app/components/workflow-app/hooks/use-workflow-run.ts @@ -20,7 +20,8 @@ import type { VersionHistory } from '@/types/workflow' import { noop } from 'lodash-es' import { useNodesSyncDraft } from './use-nodes-sync-draft' import { useInvalidAllLastRun } from '@/service/use-workflow' -import { useSetWorkflowVarsWithValue } from './use-fetch-workflow-inspect-vars' +import { useSetWorkflowVarsWithValue } from '../../workflow/hooks/use-fetch-workflow-inspect-vars' +import { useConfigsMap } from './use-configs-map' export const useWorkflowRun = () => { const store = useStoreApi() @@ -32,7 +33,11 @@ export const useWorkflowRun = () => { const pathname = usePathname() const appId = useAppStore.getState().appDetail?.id const invalidAllLastRun = useInvalidAllLastRun(appId as string) - const { fetchInspectVars } = useSetWorkflowVarsWithValue() + const configsMap = useConfigsMap() + const { fetchInspectVars } = useSetWorkflowVarsWithValue({ + flowId: appId as string, + ...configsMap, + }) const { handleWorkflowStarted, diff --git a/web/app/components/workflow/block-selector/all-tools.tsx b/web/app/components/workflow/block-selector/all-tools.tsx index e57a6bd3f7..870d791d4f 100644 --- a/web/app/components/workflow/block-selector/all-tools.tsx +++ b/web/app/components/workflow/block-selector/all-tools.tsx @@ -5,10 +5,11 @@ import { useState, } from 'react' import type { + BlockEnum, OnSelectBlock, ToolWithProvider, } from '../types' -import type { ToolValue } from './types' +import type { ToolDefaultValue, ToolValue } from './types' import { ToolTypeEnum } from './types' import Tools from './tools' import { useToolTabs } from './hooks' @@ -17,8 +18,6 @@ import cn from '@/utils/classnames' import { useGetLanguage } from '@/context/i18n' import type { ListRef } from '@/app/components/workflow/block-selector/market-place-plugin/list' import PluginList, { type ListProps } from '@/app/components/workflow/block-selector/market-place-plugin/list' -import ActionButton from '../../base/action-button' -import { RiAddLine } from '@remixicon/react' import { PluginType } from '../../plugins/types' import { useMarketplacePlugins } from '../../plugins/marketplace/hooks' import { useGlobalPublicStore } from '@/context/global-public-context' @@ -31,11 +30,12 @@ type AllToolsProps = { buildInTools: ToolWithProvider[] customTools: ToolWithProvider[] workflowTools: ToolWithProvider[] + mcpTools: ToolWithProvider[] onSelect: OnSelectBlock - supportAddCustomTool?: boolean - onAddedCustomTool?: () => void - onShowAddCustomCollectionModal?: () => void + canNotSelectMultiple?: boolean + onSelectMultiple?: (type: BlockEnum, tools: ToolDefaultValue[]) => void selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const DEFAULT_TAGS: AllToolsProps['tags'] = [] @@ -46,12 +46,14 @@ const AllTools = ({ searchText, tags = DEFAULT_TAGS, onSelect, + canNotSelectMultiple, + onSelectMultiple, buildInTools, workflowTools, customTools, - supportAddCustomTool, - onShowAddCustomCollectionModal, + mcpTools = [], selectedTools, + canChooseMCPTool, }: AllToolsProps) => { const language = useGetLanguage() const tabs = useToolTabs() @@ -64,13 +66,15 @@ const AllTools = ({ const tools = useMemo(() => { let mergedTools: ToolWithProvider[] = [] if (activeTab === ToolTypeEnum.All) - mergedTools = [...buildInTools, ...customTools, ...workflowTools] + mergedTools = [...buildInTools, ...customTools, ...workflowTools, ...mcpTools] if (activeTab === ToolTypeEnum.BuiltIn) mergedTools = buildInTools if (activeTab === ToolTypeEnum.Custom) mergedTools = customTools if (activeTab === ToolTypeEnum.Workflow) mergedTools = workflowTools + if (activeTab === ToolTypeEnum.MCP) + mergedTools = mcpTools if (!hasFilter) return mergedTools.filter(toolWithProvider => toolWithProvider.tools.length > 0) @@ -80,7 +84,7 @@ const AllTools = ({ return tool.label[language].toLowerCase().includes(searchText.toLowerCase()) || tool.name.toLowerCase().includes(searchText.toLowerCase()) }) }) - }, [activeTab, buildInTools, customTools, workflowTools, searchText, language, hasFilter]) + }, [activeTab, buildInTools, customTools, workflowTools, mcpTools, searchText, language, hasFilter]) const { queryPluginsWithDebounced: fetchPlugins, @@ -88,7 +92,6 @@ const AllTools = ({ } = useMarketplacePlugins() const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) - useEffect(() => { if (!enable_marketplace) return if (searchText || tags.length > 0) { @@ -103,10 +106,11 @@ const AllTools = ({ const pluginRef = useRef(null) const wrapElemRef = useRef(null) + const isSupportGroupView = [ToolTypeEnum.All, ToolTypeEnum.BuiltIn].includes(activeTab) return ( -
-
+
+
{ tabs.map(tab => ( @@ -124,17 +128,8 @@ const AllTools = ({ )) }
- - {supportAddCustomTool && ( -
-
- - - -
+ {isSupportGroupView && ( + )}
{/* Plugins from marketplace */} {enable_marketplace && { ] } -export const useToolTabs = () => { +export const useToolTabs = (isHideMCPTools?: boolean) => { const { t } = useTranslation() - - return [ + const tabs = [ { key: ToolTypeEnum.All, name: t('workflow.tabs.allTool'), @@ -52,4 +51,12 @@ export const useToolTabs = () => { name: t('workflow.tabs.workflowTool'), }, ] + if(!isHideMCPTools) { + tabs.push({ + key: ToolTypeEnum.MCP, + name: 'MCP', + }) + } + + return tabs } diff --git a/web/app/components/workflow/block-selector/index-bar.tsx b/web/app/components/workflow/block-selector/index-bar.tsx index 4d8bedffbe..097a16eb94 100644 --- a/web/app/components/workflow/block-selector/index-bar.tsx +++ b/web/app/components/workflow/block-selector/index-bar.tsx @@ -83,8 +83,8 @@ const IndexBar: FC = ({ letters, itemRefs, className }) => { element.scrollIntoView({ behavior: 'smooth' }) } return ( -
-
+
+
{letters.map(letter => (
handleIndexClick(letter)}> {letter} diff --git a/web/app/components/workflow/block-selector/index.tsx b/web/app/components/workflow/block-selector/index.tsx index 9e55a24d9e..0673ca0c0d 100644 --- a/web/app/components/workflow/block-selector/index.tsx +++ b/web/app/components/workflow/block-selector/index.tsx @@ -129,33 +129,35 @@ const NodeSelector: FC = ({
-
e.stopPropagation()}> - {activeTab === TabsEnum.Blocks && ( - setSearchText(e.target.value)} - onClear={() => setSearchText('')} - /> - )} - {activeTab === TabsEnum.Tools && ( - - )} - -
e.stopPropagation()}> + {activeTab === TabsEnum.Blocks && ( + setSearchText(e.target.value)} + onClear={() => setSearchText('')} + /> + )} + {activeTab === TabsEnum.Tools && ( + + )} +
+ } onSelect={handleSelect} searchText={searchText} tags={tags} diff --git a/web/app/components/workflow/block-selector/market-place-plugin/list.tsx b/web/app/components/workflow/block-selector/market-place-plugin/list.tsx index e2b4a7acc6..dce877ab91 100644 --- a/web/app/components/workflow/block-selector/market-place-plugin/list.tsx +++ b/web/app/components/workflow/block-selector/market-place-plugin/list.tsx @@ -80,7 +80,7 @@ const List = forwardRef(({ ) } - const maxWidthClassName = toolContentClassName || 'max-w-[300px]' + const maxWidthClassName = toolContentClassName || 'max-w-[100%]' return ( <> @@ -109,18 +109,20 @@ const List = forwardRef(({ onAction={noop} /> ))} -
-
- - - {t('plugin.searchInMarketplace')} - -
-
+ {list.length > 0 && ( +
+
+ + + {t('plugin.searchInMarketplace')} + +
+
+ )}
) diff --git a/web/app/components/workflow/block-selector/tabs.tsx b/web/app/components/workflow/block-selector/tabs.tsx index 67aaaba1a5..3f3fed2ca9 100644 --- a/web/app/components/workflow/block-selector/tabs.tsx +++ b/web/app/components/workflow/block-selector/tabs.tsx @@ -1,6 +1,6 @@ import type { FC } from 'react' import { memo } from 'react' -import { useAllBuiltInTools, useAllCustomTools, useAllWorkflowTools } from '@/service/use-tools' +import { useAllBuiltInTools, useAllCustomTools, useAllMCPTools, useAllWorkflowTools } from '@/service/use-tools' import type { BlockEnum } from '../types' import { useTabs } from './hooks' import type { ToolDefaultValue } from './types' @@ -16,6 +16,7 @@ export type TabsProps = { tags: string[] onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void availableBlocksTypes?: BlockEnum[] + filterElem: React.ReactNode noBlocks?: boolean } const Tabs: FC = ({ @@ -25,26 +26,28 @@ const Tabs: FC = ({ searchText, onSelect, availableBlocksTypes, + filterElem, noBlocks, }) => { const tabs = useTabs() const { data: buildInTools } = useAllBuiltInTools() const { data: customTools } = useAllCustomTools() const { data: workflowTools } = useAllWorkflowTools() + const { data: mcpTools } = useAllMCPTools() return (
e.stopPropagation()}> { !noBlocks && ( -
+
{ tabs.map(tab => (
onActiveTabChange(tab.key)} @@ -56,25 +59,30 @@ const Tabs: FC = ({
) } + {filterElem} { activeTab === TabsEnum.Blocks && !noBlocks && ( - +
+ +
) } { activeTab === TabsEnum.Tools && ( ) } diff --git a/web/app/components/workflow/block-selector/tool-picker.tsx b/web/app/components/workflow/block-selector/tool-picker.tsx index dbb49fde75..d97a4f3a1b 100644 --- a/web/app/components/workflow/block-selector/tool-picker.tsx +++ b/web/app/components/workflow/block-selector/tool-picker.tsx @@ -23,7 +23,7 @@ import { } from '@/service/tools' import type { CustomCollectionBackend } from '@/app/components/tools/types' import Toast from '@/app/components/base/toast' -import { useAllBuiltInTools, useAllCustomTools, useAllWorkflowTools, useInvalidateAllCustomTools } from '@/service/use-tools' +import { useAllBuiltInTools, useAllCustomTools, useAllMCPTools, useAllWorkflowTools, useInvalidateAllCustomTools } from '@/service/use-tools' import cn from '@/utils/classnames' type Props = { @@ -35,9 +35,11 @@ type Props = { isShow: boolean onShowChange: (isShow: boolean) => void onSelect: (tool: ToolDefaultValue) => void + onSelectMultiple: (tools: ToolDefaultValue[]) => void supportAddCustomTool?: boolean scope?: string selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const ToolPicker: FC = ({ @@ -48,10 +50,12 @@ const ToolPicker: FC = ({ isShow, onShowChange, onSelect, + onSelectMultiple, supportAddCustomTool, scope = 'all', selectedTools, panelClassName, + canChooseMCPTool, }) => { const { t } = useTranslation() const [searchText, setSearchText] = useState('') @@ -61,6 +65,7 @@ const ToolPicker: FC = ({ const { data: customTools } = useAllCustomTools() const invalidateCustomTools = useInvalidateAllCustomTools() const { data: workflowTools } = useAllWorkflowTools() + const { data: mcpTools } = useAllMCPTools() const { builtinToolList, customToolList, workflowToolList } = useMemo(() => { if (scope === 'plugins') { @@ -102,6 +107,10 @@ const ToolPicker: FC = ({ onSelect(tool!) } + const handleSelectMultiple = (_type: BlockEnum, tools: ToolDefaultValue[]) => { + onSelectMultiple(tools) + } + const [isShowEditCollectionToolModal, { setFalse: hideEditCustomCollectionModal, setTrue: showEditCustomCollectionModal, @@ -142,7 +151,7 @@ const ToolPicker: FC = ({ -
+
= ({ onTagsChange={setTags} size='small' placeholder={t('plugin.searchTools')!} + supportAddCustomTool={supportAddCustomTool} + onAddedCustomTool={handleAddedCustomTool} + onShowAddCustomCollectionModal={showEditCustomCollectionModal} + inputClassName='grow' + />
diff --git a/web/app/components/workflow/block-selector/tool/action-item.tsx b/web/app/components/workflow/block-selector/tool/action-item.tsx index dc9b9b9114..e5e33614b0 100644 --- a/web/app/components/workflow/block-selector/tool/action-item.tsx +++ b/web/app/components/workflow/block-selector/tool/action-item.tsx @@ -10,13 +10,12 @@ import { useGetLanguage } from '@/context/i18n' import BlockIcon from '../../block-icon' import cn from '@/utils/classnames' import { useTranslation } from 'react-i18next' -import { RiCheckLine } from '@remixicon/react' -import Badge from '@/app/components/base/badge' type Props = { provider: ToolWithProvider payload: Tool disabled?: boolean + isAdded?: boolean onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void } @@ -25,6 +24,7 @@ const ToolItem: FC = ({ payload, onSelect, disabled, + isAdded, }) => { const { t } = useTranslation() @@ -71,18 +71,16 @@ const ToolItem: FC = ({ output_schema: payload.output_schema, paramSchemas: payload.parameters, params, + meta: provider.meta, }) }} > -
{payload.label[language]}
- {disabled && - -
{t('tools.addToolModal.added')}
- - } +
+ {payload.label[language]} +
+ {isAdded && ( +
{t('tools.addToolModal.added')}
+ )}
) diff --git a/web/app/components/workflow/block-selector/tool/tool-list-flat-view/list.tsx b/web/app/components/workflow/block-selector/tool/tool-list-flat-view/list.tsx index ef671ca1f8..ca462c082e 100644 --- a/web/app/components/workflow/block-selector/tool/tool-list-flat-view/list.tsx +++ b/web/app/components/workflow/block-selector/tool/tool-list-flat-view/list.tsx @@ -11,21 +11,29 @@ import { useMemo } from 'react' type Props = { payload: ToolWithProvider[] isShowLetterIndex: boolean + indexBar: React.ReactNode hasSearchText: boolean onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void + canNotSelectMultiple?: boolean + onSelectMultiple?: (type: BlockEnum, tools: ToolDefaultValue[]) => void letters: string[] toolRefs: any selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const ToolViewFlatView: FC = ({ letters, payload, isShowLetterIndex, + indexBar, hasSearchText, onSelect, + canNotSelectMultiple, + onSelectMultiple, toolRefs, selectedTools, + canChooseMCPTool, }) => { const firstLetterToolIds = useMemo(() => { const res: Record = {} @@ -37,26 +45,31 @@ const ToolViewFlatView: FC = ({ return res }, [payload, letters]) return ( -
- {payload.map(tool => ( -
{ - const letter = firstLetterToolIds[tool.id] - if (letter) - toolRefs.current[letter] = el - }} - > - -
- ))} +
+
+ {payload.map(tool => ( +
{ + const letter = firstLetterToolIds[tool.id] + if (letter) + toolRefs.current[letter] = el + }} + > + +
+ ))} +
+ {isShowLetterIndex && indexBar}
) } diff --git a/web/app/components/workflow/block-selector/tool/tool-list-tree-view/item.tsx b/web/app/components/workflow/block-selector/tool/tool-list-tree-view/item.tsx index d6c567f8e2..b3f7aab4df 100644 --- a/web/app/components/workflow/block-selector/tool/tool-list-tree-view/item.tsx +++ b/web/app/components/workflow/block-selector/tool/tool-list-tree-view/item.tsx @@ -12,7 +12,10 @@ type Props = { toolList: ToolWithProvider[] hasSearchText: boolean onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void + canNotSelectMultiple?: boolean + onSelectMultiple?: (type: BlockEnum, tools: ToolDefaultValue[]) => void selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const Item: FC = ({ @@ -20,7 +23,10 @@ const Item: FC = ({ toolList, hasSearchText, onSelect, + canNotSelectMultiple, + onSelectMultiple, selectedTools, + canChooseMCPTool, }) => { return (
@@ -36,7 +42,10 @@ const Item: FC = ({ isShowLetterIndex={false} hasSearchText={hasSearchText} onSelect={onSelect} + canNotSelectMultiple={canNotSelectMultiple} + onSelectMultiple={onSelectMultiple} selectedTools={selectedTools} + canChooseMCPTool={canChooseMCPTool} /> ))}
diff --git a/web/app/components/workflow/block-selector/tool/tool-list-tree-view/list.tsx b/web/app/components/workflow/block-selector/tool/tool-list-tree-view/list.tsx index f3f98279c8..d85d1ea682 100644 --- a/web/app/components/workflow/block-selector/tool/tool-list-tree-view/list.tsx +++ b/web/app/components/workflow/block-selector/tool/tool-list-tree-view/list.tsx @@ -12,14 +12,20 @@ type Props = { payload: Record hasSearchText: boolean onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void + canNotSelectMultiple?: boolean + onSelectMultiple?: (type: BlockEnum, tools: ToolDefaultValue[]) => void selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const ToolListTreeView: FC = ({ payload, hasSearchText, onSelect, + canNotSelectMultiple, + onSelectMultiple, selectedTools, + canChooseMCPTool, }) => { const { t } = useTranslation() const getI18nGroupName = useCallback((name: string) => { @@ -46,7 +52,10 @@ const ToolListTreeView: FC = ({ toolList={payload[groupName]} hasSearchText={hasSearchText} onSelect={onSelect} + canNotSelectMultiple={canNotSelectMultiple} + onSelectMultiple={onSelectMultiple} selectedTools={selectedTools} + canChooseMCPTool={canChooseMCPTool} /> ))}
diff --git a/web/app/components/workflow/block-selector/tool/tool.tsx b/web/app/components/workflow/block-selector/tool/tool.tsx index d48d0bfc90..83ae062737 100644 --- a/web/app/components/workflow/block-selector/tool/tool.tsx +++ b/web/app/components/workflow/block-selector/tool/tool.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useEffect, useMemo } from 'react' +import React, { useCallback, useEffect, useMemo, useRef } from 'react' import cn from '@/utils/classnames' import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react' import { useGetLanguage } from '@/context/i18n' @@ -13,36 +13,108 @@ import { ViewType } from '../view-type-select' import ActionItem from './action-item' import BlockIcon from '../../block-icon' import { useTranslation } from 'react-i18next' +import { useHover } from 'ahooks' +import McpToolNotSupportTooltip from '../../nodes/_base/components/mcp-tool-not-support-tooltip' +import { Mcp } from '@/app/components/base/icons/src/vender/other' type Props = { className?: string payload: ToolWithProvider viewType: ViewType - isShowLetterIndex: boolean hasSearchText: boolean onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void + canNotSelectMultiple?: boolean + onSelectMultiple?: (type: BlockEnum, tools: ToolDefaultValue[]) => void selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const Tool: FC = ({ className, payload, viewType, - isShowLetterIndex, hasSearchText, onSelect, + canNotSelectMultiple, + onSelectMultiple, selectedTools, + canChooseMCPTool, }) => { const { t } = useTranslation() const language = useGetLanguage() const isFlatView = viewType === ViewType.flat + const notShowProvider = payload.type === CollectionType.workflow const actions = payload.tools - const hasAction = true // Now always support actions + const hasAction = !notShowProvider const [isFold, setFold] = React.useState(true) - const getIsDisabled = (tool: ToolType) => { + const ref = useRef(null) + const isHovering = useHover(ref) + const isMCPTool = payload.type === CollectionType.mcp + const isShowCanNotChooseMCPTip = !canChooseMCPTool && isMCPTool + const getIsDisabled = useCallback((tool: ToolType) => { if (!selectedTools || !selectedTools.length) return false - return selectedTools.some(selectedTool => selectedTool.provider_name === payload.name && selectedTool.tool_name === tool.name) - } + return selectedTools.some(selectedTool => (selectedTool.provider_name === payload.name || selectedTool.provider_name === payload.id) && selectedTool.tool_name === tool.name) + }, [payload.id, payload.name, selectedTools]) + + const totalToolsNum = actions.length + const selectedToolsNum = actions.filter(action => getIsDisabled(action)).length + const isAllSelected = selectedToolsNum === totalToolsNum + + const notShowProviderSelectInfo = useMemo(() => { + if (isAllSelected) { + return ( + + {t('tools.addToolModal.added')} + + ) + } + }, [isAllSelected, t]) + const selectedInfo = useMemo(() => { + if (isHovering && !isAllSelected) { + return ( + { + onSelectMultiple?.(BlockEnum.Tool, actions.filter(action => !getIsDisabled(action)).map((tool) => { + const params: Record = {} + if (tool.parameters) { + tool.parameters.forEach((item) => { + params[item.name] = '' + }) + } + return { + provider_id: payload.id, + provider_type: payload.type, + provider_name: payload.name, + tool_name: tool.name, + tool_label: tool.label[language], + tool_description: tool.description[language], + title: tool.label[language], + is_team_authorization: payload.is_team_authorization, + output_schema: tool.output_schema, + paramSchemas: tool.parameters, + params, + } + })) + }} + > + {t('workflow.tabs.addAll')} + + ) + } + + if (selectedToolsNum === 0) + return <> + + return ( + + {isAllSelected + ? t('workflow.tabs.allAdded') + : `${selectedToolsNum} / ${totalToolsNum}` + } + + ) + }, [actions, getIsDisabled, isAllSelected, isHovering, language, onSelectMultiple, payload.id, payload.is_team_authorization, payload.name, payload.type, selectedToolsNum, t, totalToolsNum]) + useEffect(() => { if (hasSearchText && isFold) { setFold(false) @@ -71,59 +143,73 @@ const Tool: FC = ({ return (
{ - if (hasAction) + if (hasAction) { setFold(!isFold) + return + } - // Now always support actions - // if (payload.parameters) { - // payload.parameters.forEach((item) => { - // params[item.name] = '' - // }) - // } - // onSelect(BlockEnum.Tool, { - // provider_id: payload.id, - // provider_type: payload.type, - // provider_name: payload.name, - // tool_name: payload.name, - // tool_label: payload.label[language], - // title: payload.label[language], - // params: {}, - // }) + const tool = actions[0] + const params: Record = {} + if (tool.parameters) { + tool.parameters.forEach((item) => { + params[item.name] = '' + }) + } + onSelect(BlockEnum.Tool, { + provider_id: payload.id, + provider_type: payload.type, + provider_name: payload.name, + tool_name: tool.name, + tool_label: tool.label[language], + tool_description: tool.description[language], + title: tool.label[language], + is_team_authorization: payload.is_team_authorization, + output_schema: tool.output_schema, + paramSchemas: tool.parameters, + params, + }) }} > -
+
-
{payload.label[language]}
+
+ {notShowProvider ? actions[0]?.label[language] : payload.label[language]} + {isFlatView && groupName && ( + {groupName} + )} + {isMCPTool && } +
-
- {isFlatView && ( -
{groupName}
- )} +
+ {!isShowCanNotChooseMCPTip && !canNotSelectMultiple && (notShowProvider ? notShowProviderSelectInfo : selectedInfo)} + {isShowCanNotChooseMCPTip && } {hasAction && ( - + )}
- {hasAction && !isFold && ( + {!notShowProvider && hasAction && !isFold && ( actions.map(action => ( )) )} diff --git a/web/app/components/workflow/block-selector/tools.tsx b/web/app/components/workflow/block-selector/tools.tsx index 2562501524..da47432b04 100644 --- a/web/app/components/workflow/block-selector/tools.tsx +++ b/web/app/components/workflow/block-selector/tools.tsx @@ -7,6 +7,7 @@ import { useTranslation } from 'react-i18next' import type { BlockEnum, ToolWithProvider } from '../types' import IndexBar, { groupItems } from './index-bar' import type { ToolDefaultValue, ToolValue } from './types' +import type { ToolTypeEnum } from './types' import { ViewType } from './view-type-select' import Empty from '@/app/components/tools/add-tool-modal/empty' import { useGetLanguage } from '@/context/i18n' @@ -15,25 +16,34 @@ import ToolListFlatView from './tool/tool-list-flat-view/list' import classNames from '@/utils/classnames' type ToolsProps = { - showWorkflowEmpty: boolean onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void + canNotSelectMultiple?: boolean + onSelectMultiple?: (type: BlockEnum, tools: ToolDefaultValue[]) => void tools: ToolWithProvider[] viewType: ViewType hasSearchText: boolean + toolType?: ToolTypeEnum + isAgent?: boolean className?: string indexBarClassName?: string selectedTools?: ToolValue[] + canChooseMCPTool?: boolean } const Blocks = ({ - showWorkflowEmpty, onSelect, + canNotSelectMultiple, + onSelectMultiple, tools, viewType, hasSearchText, + toolType, + isAgent, className, indexBarClassName, selectedTools, + canChooseMCPTool, }: ToolsProps) => { + // const tools: any = [] const { t } = useTranslation() const language = useGetLanguage() const isFlatView = viewType === ViewType.flat @@ -87,15 +97,15 @@ const Blocks = ({ const toolRefs = useRef({}) return ( -
+
{ - !tools.length && !showWorkflowEmpty && ( -
{t('workflow.tabs.noResult')}
+ !tools.length && hasSearchText && ( +
{t('workflow.tabs.noResult')}
) } - {!tools.length && showWorkflowEmpty && ( + {!tools.length && !hasSearchText && (
- +
)} {!!tools.length && ( @@ -107,19 +117,24 @@ const Blocks = ({ isShowLetterIndex={isShowLetterIndex} hasSearchText={hasSearchText} onSelect={onSelect} + canNotSelectMultiple={canNotSelectMultiple} + onSelectMultiple={onSelectMultiple} selectedTools={selectedTools} + canChooseMCPTool={canChooseMCPTool} + indexBar={} /> ) : ( ) )} - - {isShowLetterIndex && }
) } diff --git a/web/app/components/workflow/block-selector/types.ts b/web/app/components/workflow/block-selector/types.ts index f1bdbbfbd9..c96a60f674 100644 --- a/web/app/components/workflow/block-selector/types.ts +++ b/web/app/components/workflow/block-selector/types.ts @@ -1,3 +1,5 @@ +import type { PluginMeta } from '../../plugins/types' + export enum TabsEnum { Blocks = 'blocks', Tools = 'tools', @@ -8,6 +10,7 @@ export enum ToolTypeEnum { BuiltIn = 'built-in', Custom = 'custom', Workflow = 'workflow', + MCP = 'mcp', } export enum BlockClassificationEnum { @@ -30,10 +33,12 @@ export type ToolDefaultValue = { params: Record paramSchemas: Record[] output_schema: Record + meta?: PluginMeta } export type ToolValue = { provider_name: string + provider_show_name?: string tool_name: string tool_label: string tool_description?: string diff --git a/web/app/components/workflow/block-selector/use-check-vertical-scrollbar.ts b/web/app/components/workflow/block-selector/use-check-vertical-scrollbar.ts new file mode 100644 index 0000000000..98986cf3b6 --- /dev/null +++ b/web/app/components/workflow/block-selector/use-check-vertical-scrollbar.ts @@ -0,0 +1,31 @@ +import { useEffect, useState } from 'react' + +const useCheckVerticalScrollbar = (ref: React.RefObject) => { + const [hasVerticalScrollbar, setHasVerticalScrollbar] = useState(false) + + useEffect(() => { + const elem = ref.current + if (!elem) return + + const checkScrollbar = () => { + setHasVerticalScrollbar(elem.scrollHeight > elem.clientHeight) + } + + checkScrollbar() + + const resizeObserver = new ResizeObserver(checkScrollbar) + resizeObserver.observe(elem) + + const mutationObserver = new MutationObserver(checkScrollbar) + mutationObserver.observe(elem, { childList: true, subtree: true, characterData: true }) + + return () => { + resizeObserver.disconnect() + mutationObserver.disconnect() + } + }, [ref]) + + return hasVerticalScrollbar +} + +export default useCheckVerticalScrollbar diff --git a/web/app/components/workflow-app/hooks/use-fetch-workflow-inspect-vars.ts b/web/app/components/workflow/hooks/use-fetch-workflow-inspect-vars.ts similarity index 85% rename from web/app/components/workflow-app/hooks/use-fetch-workflow-inspect-vars.ts rename to web/app/components/workflow/hooks/use-fetch-workflow-inspect-vars.ts index 07580c097e..27a9ea9d2d 100644 --- a/web/app/components/workflow-app/hooks/use-fetch-workflow-inspect-vars.ts +++ b/web/app/components/workflow/hooks/use-fetch-workflow-inspect-vars.ts @@ -6,12 +6,20 @@ import type { Node } from '@/app/components/workflow/types' import { fetchAllInspectVars } from '@/service/workflow' import { useInvalidateConversationVarValues, useInvalidateSysVarValues } from '@/service/use-workflow' import { useNodesInteractionsWithoutSync } from '@/app/components/workflow/hooks/use-nodes-interactions-without-sync' -import { useConfigsMap } from './use-configs-map' -export const useSetWorkflowVarsWithValue = () => { +type Params = { + flowId: string + conversationVarsUrl: string + systemVarsUrl: string +} + +export const useSetWorkflowVarsWithValue = ({ + flowId, + conversationVarsUrl, + systemVarsUrl, +}: Params) => { const workflowStore = useWorkflowStore() const store = useStoreApi() - const { conversationVarsUrl, systemVarsUrl } = useConfigsMap() const invalidateConversationVarValues = useInvalidateConversationVarValues(conversationVarsUrl) const invalidateSysVarValues = useInvalidateSysVarValues(systemVarsUrl) const { handleCancelAllNodeSuccessStatus } = useNodesInteractionsWithoutSync() @@ -58,13 +66,12 @@ export const useSetWorkflowVarsWithValue = () => { }, [workflowStore, store]) const fetchInspectVars = useCallback(async () => { - const { appId } = workflowStore.getState() invalidateConversationVarValues() invalidateSysVarValues() - const data = await fetchAllInspectVars(appId) + const data = await fetchAllInspectVars(flowId) setInspectVarsToStore(data) handleCancelAllNodeSuccessStatus() // to make sure clear node output show the unset status - }, [workflowStore, invalidateConversationVarValues, invalidateSysVarValues, setInspectVarsToStore, handleCancelAllNodeSuccessStatus]) + }, [invalidateConversationVarValues, invalidateSysVarValues, flowId, setInspectVarsToStore, handleCancelAllNodeSuccessStatus]) return { fetchInspectVars, } diff --git a/web/app/components/workflow/hooks/use-inspect-vars-crud-common.ts b/web/app/components/workflow/hooks/use-inspect-vars-crud-common.ts new file mode 100644 index 0000000000..ffcfd81666 --- /dev/null +++ b/web/app/components/workflow/hooks/use-inspect-vars-crud-common.ts @@ -0,0 +1,240 @@ +import { fetchNodeInspectVars } from '@/service/workflow' +import { useWorkflowStore } from '@/app/components/workflow/store' +import type { ValueSelector } from '@/app/components/workflow/types' +import type { VarInInspect } from '@/types/workflow' +import { VarInInspectType } from '@/types/workflow' +import { + useDeleteAllInspectorVars, + useDeleteInspectVar, + useDeleteNodeInspectorVars, + useEditInspectorVar, + useInvalidateConversationVarValues, + useInvalidateSysVarValues, + useResetConversationVar, + useResetToLastRunValue, +} from '@/service/use-workflow' +import { useCallback } from 'react' +import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import produce from 'immer' +import type { Node } from '@/app/components/workflow/types' +import { useNodesInteractionsWithoutSync } from '@/app/components/workflow/hooks/use-nodes-interactions-without-sync' +import { useEdgesInteractionsWithoutSync } from '@/app/components/workflow/hooks/use-edges-interactions-without-sync' + +type Params = { + flowId: string + conversationVarsUrl: string + systemVarsUrl: string +} +export const useInspectVarsCrudCommon = ({ + flowId, + conversationVarsUrl, + systemVarsUrl, +}: Params) => { + const workflowStore = useWorkflowStore() + const invalidateConversationVarValues = useInvalidateConversationVarValues(conversationVarsUrl!) + const { mutateAsync: doResetConversationVar } = useResetConversationVar(flowId) + const { mutateAsync: doResetToLastRunValue } = useResetToLastRunValue(flowId) + const invalidateSysVarValues = useInvalidateSysVarValues(systemVarsUrl!) + + const { mutateAsync: doDeleteAllInspectorVars } = useDeleteAllInspectorVars(flowId) + const { mutate: doDeleteNodeInspectorVars } = useDeleteNodeInspectorVars(flowId) + const { mutate: doDeleteInspectVar } = useDeleteInspectVar(flowId) + + const { mutateAsync: doEditInspectorVar } = useEditInspectorVar(flowId) + const { handleCancelNodeSuccessStatus } = useNodesInteractionsWithoutSync() + const { handleEdgeCancelRunningStatus } = useEdgesInteractionsWithoutSync() + const getNodeInspectVars = useCallback((nodeId: string) => { + const { nodesWithInspectVars } = workflowStore.getState() + const node = nodesWithInspectVars.find(node => node.nodeId === nodeId) + return node + }, [workflowStore]) + + const getVarId = useCallback((nodeId: string, varName: string) => { + const node = getNodeInspectVars(nodeId) + if (!node) + return undefined + const varId = node.vars.find((varItem) => { + return varItem.selector[1] === varName + })?.id + return varId + }, [getNodeInspectVars]) + + const getInspectVar = useCallback((nodeId: string, name: string): VarInInspect | undefined => { + const node = getNodeInspectVars(nodeId) + if (!node) + return undefined + + const variable = node.vars.find((varItem) => { + return varItem.name === name + }) + return variable + }, [getNodeInspectVars]) + + const hasSetInspectVar = useCallback((nodeId: string, name: string, sysVars: VarInInspect[], conversationVars: VarInInspect[]) => { + const isEnv = isENV([nodeId]) + if (isEnv) // always have value + return true + const isSys = isSystemVar([nodeId]) + if (isSys) + return sysVars.some(varItem => varItem.selector?.[1]?.replace('sys.', '') === name) + const isChatVar = isConversationVar([nodeId]) + if (isChatVar) + return conversationVars.some(varItem => varItem.selector?.[1] === name) + return getInspectVar(nodeId, name) !== undefined + }, [getInspectVar]) + + const hasNodeInspectVars = useCallback((nodeId: string) => { + return !!getNodeInspectVars(nodeId) + }, [getNodeInspectVars]) + + const fetchInspectVarValue = useCallback(async (selector: ValueSelector) => { + const { + appId, + setNodeInspectVars, + } = workflowStore.getState() + const nodeId = selector[0] + const isSystemVar = nodeId === 'sys' + const isConversationVar = nodeId === 'conversation' + if (isSystemVar) { + invalidateSysVarValues() + return + } + if (isConversationVar) { + invalidateConversationVarValues() + return + } + const vars = await fetchNodeInspectVars(appId, nodeId) + setNodeInspectVars(nodeId, vars) + }, [workflowStore, invalidateSysVarValues, invalidateConversationVarValues]) + + // after last run would call this + const appendNodeInspectVars = useCallback((nodeId: string, payload: VarInInspect[], allNodes: Node[]) => { + const { + nodesWithInspectVars, + setNodesWithInspectVars, + } = workflowStore.getState() + const nodes = produce(nodesWithInspectVars, (draft) => { + const nodeInfo = allNodes.find(node => node.id === nodeId) + if (nodeInfo) { + const index = draft.findIndex(node => node.nodeId === nodeId) + if (index === -1) { + draft.unshift({ + nodeId, + nodeType: nodeInfo.data.type, + title: nodeInfo.data.title, + vars: payload, + nodePayload: nodeInfo.data, + }) + } + else { + draft[index].vars = payload + // put the node to the topAdd commentMore actions + draft.unshift(draft.splice(index, 1)[0]) + } + } + }) + setNodesWithInspectVars(nodes) + handleCancelNodeSuccessStatus(nodeId) + }, [workflowStore, handleCancelNodeSuccessStatus]) + + const hasNodeInspectVar = useCallback((nodeId: string, varId: string) => { + const { nodesWithInspectVars } = workflowStore.getState() + const targetNode = nodesWithInspectVars.find(item => item.nodeId === nodeId) + if(!targetNode || !targetNode.vars) + return false + return targetNode.vars.some(item => item.id === varId) + }, [workflowStore]) + + const deleteInspectVar = useCallback(async (nodeId: string, varId: string) => { + const { deleteInspectVar } = workflowStore.getState() + if(hasNodeInspectVar(nodeId, varId)) { + await doDeleteInspectVar(varId) + deleteInspectVar(nodeId, varId) + } + }, [doDeleteInspectVar, workflowStore, hasNodeInspectVar]) + + const resetConversationVar = useCallback(async (varId: string) => { + await doResetConversationVar(varId) + invalidateConversationVarValues() + }, [doResetConversationVar, invalidateConversationVarValues]) + + const deleteNodeInspectorVars = useCallback(async (nodeId: string) => { + const { deleteNodeInspectVars } = workflowStore.getState() + if (hasNodeInspectVars(nodeId)) { + await doDeleteNodeInspectorVars(nodeId) + deleteNodeInspectVars(nodeId) + } + }, [doDeleteNodeInspectorVars, workflowStore, hasNodeInspectVars]) + + const deleteAllInspectorVars = useCallback(async () => { + const { deleteAllInspectVars } = workflowStore.getState() + await doDeleteAllInspectorVars() + await invalidateConversationVarValues() + await invalidateSysVarValues() + deleteAllInspectVars() + handleEdgeCancelRunningStatus() + }, [doDeleteAllInspectorVars, invalidateConversationVarValues, invalidateSysVarValues, workflowStore, handleEdgeCancelRunningStatus]) + + const editInspectVarValue = useCallback(async (nodeId: string, varId: string, value: any) => { + const { setInspectVarValue } = workflowStore.getState() + await doEditInspectorVar({ + varId, + value, + }) + setInspectVarValue(nodeId, varId, value) + if (nodeId === VarInInspectType.conversation) + invalidateConversationVarValues() + if (nodeId === VarInInspectType.system) + invalidateSysVarValues() + }, [doEditInspectorVar, invalidateConversationVarValues, invalidateSysVarValues, workflowStore]) + + const renameInspectVarName = useCallback(async (nodeId: string, oldName: string, newName: string) => { + const { renameInspectVarName } = workflowStore.getState() + const varId = getVarId(nodeId, oldName) + if (!varId) + return + + const newSelector = [nodeId, newName] + await doEditInspectorVar({ + varId, + name: newName, + }) + renameInspectVarName(nodeId, varId, newSelector) + }, [doEditInspectorVar, getVarId, workflowStore]) + + const isInspectVarEdited = useCallback((nodeId: string, name: string) => { + const inspectVar = getInspectVar(nodeId, name) + if (!inspectVar) + return false + + return inspectVar.edited + }, [getInspectVar]) + + const resetToLastRunVar = useCallback(async (nodeId: string, varId: string) => { + const { resetToLastRunVar } = workflowStore.getState() + const isSysVar = nodeId === 'sys' + const data = await doResetToLastRunValue(varId) + + if(isSysVar) + invalidateSysVarValues() + else + resetToLastRunVar(nodeId, varId, data.value) + }, [doResetToLastRunValue, invalidateSysVarValues, workflowStore]) + + return { + hasNodeInspectVars, + hasSetInspectVar, + fetchInspectVarValue, + editInspectVarValue, + renameInspectVarName, + appendNodeInspectVars, + deleteInspectVar, + deleteNodeInspectorVars, + deleteAllInspectorVars, + isInspectVarEdited, + resetToLastRunVar, + invalidateSysVarValues, + resetConversationVar, + invalidateConversationVarValues, + } +} diff --git a/web/app/components/workflow/hooks/use-workflow.ts b/web/app/components/workflow/hooks/use-workflow.ts index 1b98178152..8bc9d3436f 100644 --- a/web/app/components/workflow/hooks/use-workflow.ts +++ b/web/app/components/workflow/hooks/use-workflow.ts @@ -40,6 +40,7 @@ import { useStore as useAppStore } from '@/app/components/app/store' import { fetchAllBuiltInTools, fetchAllCustomTools, + fetchAllMCPTools, fetchAllWorkflowTools, } from '@/service/tools' import { CollectionType } from '@/app/components/tools/types' @@ -445,6 +446,13 @@ export const useFetchToolsData = () => { workflowTools: workflowTools || [], }) } + if(type === 'mcp') { + const mcpTools = await fetchAllMCPTools() + + workflowStore.setState({ + mcpTools: mcpTools || [], + }) + } }, [workflowStore]) return { @@ -491,6 +499,8 @@ export const useToolIcon = (data: Node['data']) => { const buildInTools = useStore(s => s.buildInTools) const customTools = useStore(s => s.customTools) const workflowTools = useStore(s => s.workflowTools) + const mcpTools = useStore(s => s.mcpTools) + const toolIcon = useMemo(() => { if(!data) return '' @@ -500,11 +510,13 @@ export const useToolIcon = (data: Node['data']) => { targetTools = buildInTools else if (data.provider_type === CollectionType.custom) targetTools = customTools + else if (data.provider_type === CollectionType.mcp) + targetTools = mcpTools else targetTools = workflowTools return targetTools.find(toolWithProvider => canFindTool(toolWithProvider.id, data.provider_id))?.icon } - }, [data, buildInTools, customTools, workflowTools]) + }, [data, buildInTools, customTools, mcpTools, workflowTools]) return toolIcon } diff --git a/web/app/components/workflow/index.tsx b/web/app/components/workflow/index.tsx index 8631eb58e3..8ea861ebb4 100644 --- a/web/app/components/workflow/index.tsx +++ b/web/app/components/workflow/index.tsx @@ -234,6 +234,7 @@ export const Workflow: FC = memo(({ handleFetchAllTools('builtin') handleFetchAllTools('custom') handleFetchAllTools('workflow') + handleFetchAllTools('mcp') }, [handleFetchAllTools]) const { diff --git a/web/app/components/workflow/nodes/_base/components/agent-strategy-selector.tsx b/web/app/components/workflow/nodes/_base/components/agent-strategy-selector.tsx index f262ae7e34..ba5281870f 100644 --- a/web/app/components/workflow/nodes/_base/components/agent-strategy-selector.tsx +++ b/web/app/components/workflow/nodes/_base/components/agent-strategy-selector.tsx @@ -68,6 +68,7 @@ function formatStrategy(input: StrategyPluginDetail[], getIcon: (i: string) => s icon: getIcon(item.declaration.identity.icon), label: item.declaration.identity.label as any, type: CollectionType.all, + meta: item.meta, tools: item.declaration.strategies.map(strategy => ({ name: strategy.identity.name, author: strategy.identity.author, @@ -89,10 +90,13 @@ function formatStrategy(input: StrategyPluginDetail[], getIcon: (i: string) => s export type AgentStrategySelectorProps = { value?: Strategy, onChange: (value?: Strategy) => void, + canChooseMCPTool: boolean, } export const AgentStrategySelector = memo((props: AgentStrategySelectorProps) => { - const { value, onChange } = props + const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) + + const { value, onChange, canChooseMCPTool } = props const [open, setOpen] = useState(false) const [viewType, setViewType] = useState(ViewType.flat) const [query, setQuery] = useState('') @@ -132,8 +136,6 @@ export const AgentStrategySelector = memo((props: AgentStrategySelectorProps) => plugins: notInstalledPlugins = [], } = useMarketplacePlugins() - const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) - useEffect(() => { if (!enable_marketplace) return if (query) { @@ -214,21 +216,25 @@ export const AgentStrategySelector = memo((props: AgentStrategySelectorProps) => agent_strategy_label: tool!.tool_label, agent_output_schema: tool!.output_schema, plugin_unique_identifier: tool!.provider_id, + meta: tool!.meta, }) setOpen(false) }} className='h-full max-h-full max-w-none overflow-y-auto' - indexBarClassName='top-0 xl:top-36' showWorkflowEmpty={false} hasSearchText={false} /> - {enable_marketplace - && + {enable_marketplace && - } + />}
diff --git a/web/app/components/workflow/nodes/_base/components/agent-strategy.tsx b/web/app/components/workflow/nodes/_base/components/agent-strategy.tsx index 4ca8746137..ce9fbb77e1 100644 --- a/web/app/components/workflow/nodes/_base/components/agent-strategy.tsx +++ b/web/app/components/workflow/nodes/_base/components/agent-strategy.tsx @@ -19,6 +19,8 @@ import { useWorkflowStore } from '../../../store' import { useRenderI18nObject } from '@/hooks/use-i18n' import type { NodeOutPutVar } from '../../../types' import type { Node } from 'reactflow' +import type { PluginMeta } from '@/app/components/plugins/types' +import { noop } from 'lodash-es' import { useDocLink } from '@/context/i18n' export type Strategy = { @@ -27,6 +29,7 @@ export type Strategy = { agent_strategy_label: string agent_output_schema: Record plugin_unique_identifier: string + meta?: PluginMeta } export type AgentStrategyProps = { @@ -38,6 +41,7 @@ export type AgentStrategyProps = { nodeOutputVars?: NodeOutPutVar[], availableNodes?: Node[], nodeId?: string + canChooseMCPTool: boolean } type CustomSchema = Omit & { type: Type } & Field @@ -48,7 +52,7 @@ type MultipleToolSelectorSchema = CustomSchema<'array[tools]'> type CustomField = ToolSelectorSchema | MultipleToolSelectorSchema export const AgentStrategy = memo((props: AgentStrategyProps) => { - const { strategy, onStrategyChange, formSchema, formValue, onFormValueChange, nodeOutputVars, availableNodes, nodeId } = props + const { strategy, onStrategyChange, formSchema, formValue, onFormValueChange, nodeOutputVars, availableNodes, nodeId, canChooseMCPTool } = props const { t } = useTranslation() const docLink = useDocLink() const defaultModel = useDefaultModel(ModelTypeEnum.textGeneration) @@ -57,6 +61,7 @@ export const AgentStrategy = memo((props: AgentStrategyProps) => { const { setControlPromptEditorRerenderKey, } = workflowStore.getState() + const override: ComponentProps>['override'] = [ [FormTypeEnum.textNumber, FormTypeEnum.textInput], (schema, props) => { @@ -168,6 +173,8 @@ export const AgentStrategy = memo((props: AgentStrategyProps) => { value={value} onSelect={item => onChange(item)} onDelete={() => onChange(null)} + canChooseMCPTool={canChooseMCPTool} + onSelectMultiple={noop} /> ) @@ -189,13 +196,14 @@ export const AgentStrategy = memo((props: AgentStrategyProps) => { onChange={onChange} supportCollapse required={schema.required} + canChooseMCPTool={canChooseMCPTool} /> ) } } } return
- + { strategy ?
@@ -215,6 +223,7 @@ export const AgentStrategy = memo((props: AgentStrategyProps) => { nodeId={nodeId} nodeOutputVars={nodeOutputVars || []} availableNodes={availableNodes || []} + canChooseMCPTool={canChooseMCPTool} />
: = ({ return ( -
+
{title}
{ diff --git a/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx b/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx index 3540c60a39..748698747c 100644 --- a/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx @@ -23,7 +23,7 @@ export type Props = { value?: string | object placeholder?: React.JSX.Element | string onChange?: (value: string) => void - title?: React.JSX.Element + title?: string | React.JSX.Element language: CodeLanguage headerRight?: React.JSX.Element readOnly?: boolean diff --git a/web/app/components/workflow/nodes/_base/components/form-input-boolean.tsx b/web/app/components/workflow/nodes/_base/components/form-input-boolean.tsx new file mode 100644 index 0000000000..07c3a087b9 --- /dev/null +++ b/web/app/components/workflow/nodes/_base/components/form-input-boolean.tsx @@ -0,0 +1,35 @@ +'use client' +import type { FC } from 'react' +import cn from '@/utils/classnames' + +type Props = { + value: boolean + onChange: (value: boolean) => void +} + +const FormInputBoolean: FC = ({ + value, + onChange, +}) => { + return ( +
+
onChange(true)} + >True
+
onChange(false)} + >False
+
+ ) +} +export default FormInputBoolean diff --git a/web/app/components/workflow/nodes/_base/components/form-input-item.tsx b/web/app/components/workflow/nodes/_base/components/form-input-item.tsx new file mode 100644 index 0000000000..316a5c9819 --- /dev/null +++ b/web/app/components/workflow/nodes/_base/components/form-input-item.tsx @@ -0,0 +1,279 @@ +'use client' +import type { FC } from 'react' +import type { ToolVarInputs } from '@/app/components/workflow/nodes/tool/types' +import type { CredentialFormSchema } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks' +import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' +import { VarType } from '@/app/components/workflow/types' + +import type { ToolWithProvider, ValueSelector, Var } from '@/app/components/workflow/types' +import FormInputTypeSwitch from './form-input-type-switch' +import useAvailableVarList from '@/app/components/workflow/nodes/_base/hooks/use-available-var-list' +import Input from '@/app/components/base/input' +import { SimpleSelect } from '@/app/components/base/select' +import MixedVariableTextInput from '@/app/components/workflow/nodes/tool/components/mixed-variable-text-input' +import FormInputBoolean from './form-input-boolean' +import AppSelector from '@/app/components/plugins/plugin-detail-panel/app-selector' +import ModelParameterModal from '@/app/components/plugins/plugin-detail-panel/model-selector' +import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker' +import CodeEditor from '@/app/components/workflow/nodes/_base/components/editor/code-editor' +import { CodeLanguage } from '@/app/components/workflow/nodes/code/types' +import cn from '@/utils/classnames' +import type { Tool } from '@/app/components/tools/types' + +type Props = { + readOnly: boolean + nodeId: string + schema: CredentialFormSchema + value: ToolVarInputs + onChange: (value: any) => void + inPanel?: boolean + currentTool?: Tool + currentProvider?: ToolWithProvider +} + +const FormInputItem: FC = ({ + readOnly, + nodeId, + schema, + value, + onChange, + inPanel, + currentTool, + currentProvider, +}) => { + const language = useLanguage() + + const { + placeholder, + variable, + type, + default: defaultValue, + options, + scope, + } = schema as any + const varInput = value[variable] + const isString = type === FormTypeEnum.textInput || type === FormTypeEnum.secretInput + const isNumber = type === FormTypeEnum.textNumber + const isObject = type === FormTypeEnum.object + const isArray = type === FormTypeEnum.array + const isShowJSONEditor = isObject || isArray + const isFile = type === FormTypeEnum.file || type === FormTypeEnum.files + const isBoolean = type === FormTypeEnum.boolean + const isSelect = type === FormTypeEnum.select || type === FormTypeEnum.dynamicSelect + const isAppSelector = type === FormTypeEnum.appSelector + const isModelSelector = type === FormTypeEnum.modelSelector + const showTypeSwitch = isNumber || isObject || isArray + const isConstant = varInput?.type === VarKindType.constant || !varInput?.type + const showVariableSelector = isFile || varInput?.type === VarKindType.variable + + const { availableVars, availableNodesWithParent } = useAvailableVarList(nodeId, { + onlyLeafNodeVar: false, + filterVar: (varPayload: Var) => { + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) + }, + }) + + const targetVarType = () => { + if (isString) + return VarType.string + else if (isNumber) + return VarType.number + else if (type === FormTypeEnum.files) + return VarType.arrayFile + else if (type === FormTypeEnum.file) + return VarType.file + // else if (isSelect) + // return VarType.select + // else if (isAppSelector) + // return VarType.appSelector + // else if (isModelSelector) + // return VarType.modelSelector + // else if (isBoolean) + // return VarType.boolean + else if (isObject) + return VarType.object + else if (isArray) + return VarType.arrayObject + else + return VarType.string + } + + const getFilterVar = () => { + if (isNumber) + return (varPayload: any) => varPayload.type === VarType.number + else if (isString) + return (varPayload: any) => [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) + else if (isFile) + return (varPayload: any) => [VarType.file, VarType.arrayFile].includes(varPayload.type) + else if (isBoolean) + return (varPayload: any) => varPayload.type === VarType.boolean + else if (isObject) + return (varPayload: any) => varPayload.type === VarType.object + else if (isArray) + return (varPayload: any) => [VarType.array, VarType.arrayString, VarType.arrayNumber, VarType.arrayObject].includes(varPayload.type) + return undefined + } + + const getVarKindType = () => { + if (isFile) + return VarKindType.variable + if (isSelect || isBoolean || isNumber || isArray || isObject) + return VarKindType.constant + if (isString) + return VarKindType.mixed + } + + const handleTypeChange = (newType: string) => { + if (newType === VarKindType.variable) { + onChange({ + ...value, + [variable]: { + ...varInput, + type: VarKindType.variable, + value: '', + }, + }) + } + else { + onChange({ + ...value, + [variable]: { + ...varInput, + type: VarKindType.constant, + value: defaultValue, + }, + }) + } + } + + const handleValueChange = (newValue: any) => { + onChange({ + ...value, + [variable]: { + ...varInput, + type: getVarKindType(), + value: isNumber ? Number.parseFloat(newValue) : newValue, + }, + }) + } + + const handleAppOrModelSelect = (newValue: any) => { + onChange({ + ...value, + [variable]: { + ...varInput, + ...newValue, + }, + }) + } + + const handleVariableSelectorChange = (newValue: ValueSelector | string, variable: string) => { + onChange({ + ...value, + [variable]: { + ...varInput, + type: VarKindType.variable, + value: newValue || '', + }, + }) + } + + return ( +
+ {showTypeSwitch && ( + + )} + {isString && ( + + )} + {isNumber && isConstant && ( + handleValueChange(e.target.value)} + placeholder={placeholder?.[language] || placeholder?.en_US} + /> + )} + {isBoolean && ( + + )} + {isSelect && ( + { + if (option.show_on.length) + return option.show_on.every(showOnItem => value[showOnItem.variable] === showOnItem.value) + + return true + }).map((option: { value: any; label: { [x: string]: any; en_US: any } }) => ({ value: option.value, name: option.label[language] || option.label.en_US }))} + onSelect={item => handleValueChange(item.value as string)} + placeholder={placeholder?.[language] || placeholder?.en_US} + /> + )} + {isShowJSONEditor && isConstant && ( +
+ {placeholder?.[language] || placeholder?.en_US}
} + /> +
+ )} + {isAppSelector && ( + + )} + {isModelSelector && isConstant && ( + + )} + {showVariableSelector && ( + handleVariableSelectorChange(value, variable)} + filterVar={getFilterVar()} + schema={schema} + valueTypePlaceHolder={targetVarType()} + currentTool={currentTool} + currentProvider={currentProvider} + /> + )} +
+ ) +} +export default FormInputItem diff --git a/web/app/components/workflow/nodes/_base/components/form-input-type-switch.tsx b/web/app/components/workflow/nodes/_base/components/form-input-type-switch.tsx new file mode 100644 index 0000000000..391e204844 --- /dev/null +++ b/web/app/components/workflow/nodes/_base/components/form-input-type-switch.tsx @@ -0,0 +1,47 @@ +'use client' +import type { FC } from 'react' +import { useTranslation } from 'react-i18next' +import { + RiEditLine, +} from '@remixicon/react' +import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import Tooltip from '@/app/components/base/tooltip' +import { VarType } from '@/app/components/workflow/nodes/tool/types' +import cn from '@/utils/classnames' + +type Props = { + value: VarType + onChange: (value: VarType) => void +} + +const FormInputTypeSwitch: FC = ({ + value, + onChange, +}) => { + const { t } = useTranslation() + return ( +
+ +
onChange(VarType.variable)} + > + +
+ + +
onChange(VarType.constant)} + > + +
+ +
+ ) +} +export default FormInputTypeSwitch diff --git a/web/app/components/workflow/nodes/_base/components/mcp-tool-not-support-tooltip.tsx b/web/app/components/workflow/nodes/_base/components/mcp-tool-not-support-tooltip.tsx new file mode 100644 index 0000000000..8117f7502f --- /dev/null +++ b/web/app/components/workflow/nodes/_base/components/mcp-tool-not-support-tooltip.tsx @@ -0,0 +1,22 @@ +'use client' +import Tooltip from '@/app/components/base/tooltip' +import { RiAlertFill } from '@remixicon/react' +import type { FC } from 'react' +import React from 'react' +import { useTranslation } from 'react-i18next' + +const McpToolNotSupportTooltip: FC = () => { + const { t } = useTranslation() + return ( + + {t('plugin.detailPanel.toolSelector.unsupportedMCPTool')} +
+ } + > + + + ) +} +export default React.memo(McpToolNotSupportTooltip) diff --git a/web/app/components/workflow/nodes/_base/components/setting-item.tsx b/web/app/components/workflow/nodes/_base/components/setting-item.tsx index 134bf4a551..abbfaef490 100644 --- a/web/app/components/workflow/nodes/_base/components/setting-item.tsx +++ b/web/app/components/workflow/nodes/_base/components/setting-item.tsx @@ -13,7 +13,7 @@ export const SettingItem = memo(({ label, children, status, tooltip }: SettingIt const indicator: ComponentProps['color'] = status === 'error' ? 'red' : status === 'warning' ? 'yellow' : undefined const needTooltip = ['error', 'warning'].includes(status as any) return
-
+
{label}
diff --git a/web/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/picker/index.tsx b/web/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/picker/index.tsx index 302ed3ca75..5a84300fcd 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/picker/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/picker/index.tsx @@ -74,7 +74,7 @@ const PickerPanel: FC = ({ ...props }) => { return ( -
+
) diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index 1058f29119..db56feaacc 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -462,6 +462,7 @@ const formatItem = ( return { variable: `env.${env.name}`, type: env.value_type, + description: env.description, } }) as Var[] break @@ -472,7 +473,7 @@ const formatItem = ( return { variable: `conversation.${chatVar.name}`, type: chatVar.value_type, - des: chatVar.description, + description: chatVar.description, } }) as Var[] break @@ -611,6 +612,7 @@ const getIterationItemType = ({ }): VarType => { const outputVarNodeId = valueSelector[0] const isSystem = isSystemVar(valueSelector) + const isChatVar = isConversationVar(valueSelector) const targetVar = isSystem ? beforeNodesOutputVars.find(v => v.isStartNode) : beforeNodesOutputVars.find(v => v.nodeId === outputVarNodeId) @@ -620,7 +622,7 @@ const getIterationItemType = ({ let arrayType: VarType = VarType.string let curr: any = targetVar.vars - if (isSystem) { + if (isSystem || isChatVar) { arrayType = curr.find((v: any) => v.variable === (valueSelector).join('.'))?.type } else { diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx index 23ccea2572..e6f3ce1fa1 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx @@ -528,6 +528,7 @@ const VarReferencePicker: FC = ({ onChange={handleVarReferenceChange} itemWidth={isAddBtnTrigger ? 260 : (minWidth || triggerWidth)} isSupportFileVar={isSupportFileVar} + zIndex={zIndex} /> )} diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-popup.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-popup.tsx index 9398ae7361..3746a85441 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-popup.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-popup.tsx @@ -13,6 +13,7 @@ type Props = { onChange: (value: ValueSelector, varDetail: Var) => void itemWidth?: number isSupportFileVar?: boolean + zIndex?: number } const VarReferencePopup: FC = ({ vars, @@ -20,6 +21,7 @@ const VarReferencePopup: FC = ({ onChange, itemWidth, isSupportFileVar = true, + zIndex, }) => { const { t } = useTranslation() const docLink = useDocLink() @@ -60,6 +62,7 @@ const VarReferencePopup: FC = ({ onChange={onChange} itemWidth={itemWidth} isSupportFileVar={isSupportFileVar} + zIndex={zIndex} /> }
diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx index 27063a2ba3..303840d8e7 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx @@ -46,6 +46,7 @@ type ItemProps = { isSupportFileVar?: boolean isException?: boolean isLoopVar?: boolean + zIndex?: number } const objVarTypes = [VarType.object, VarType.file] @@ -60,6 +61,7 @@ const Item: FC = ({ isSupportFileVar, isException, isLoopVar, + zIndex, }) => { const isStructureOutput = itemData.type === VarType.object && (itemData.children as StructuredOutput)?.schema?.properties const isFile = itemData.type === VarType.file && !isStructureOutput @@ -171,7 +173,7 @@ const Item: FC = ({
{(isStructureOutput || isObj) && ( void onBlur?: () => void + zIndex?: number autoFocus?: boolean } const VarReferenceVars: FC = ({ @@ -272,6 +275,7 @@ const VarReferenceVars: FC = ({ maxHeightClass, onClose, onBlur, + zIndex, autoFocus = true, }) => { const { t } = useTranslation() @@ -357,6 +361,7 @@ const VarReferenceVars: FC = ({ isSupportFileVar={isSupportFileVar} isException={v.isException} isLoopVar={item.isLoop} + zIndex={zIndex} /> ))}
)) diff --git a/web/app/components/workflow/nodes/_base/components/workflow-panel/index.tsx b/web/app/components/workflow/nodes/_base/components/workflow-panel/index.tsx index 164369e64c..b2a09e2f10 100644 --- a/web/app/components/workflow/nodes/_base/components/workflow-panel/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/workflow-panel/index.tsx @@ -83,18 +83,19 @@ const BasePanel: FC = ({ const otherPanelWidth = useStore(s => s.otherPanelWidth) const setNodePanelWidth = useStore(s => s.setNodePanelWidth) + const reservedCanvasWidth = 400 // Reserve the minimum visible width for the canvas + const maxNodePanelWidth = useMemo(() => { if (!workflowCanvasWidth) return 720 - if (!otherPanelWidth) - return workflowCanvasWidth - 400 - return workflowCanvasWidth - otherPanelWidth - 400 + const available = workflowCanvasWidth - (otherPanelWidth || 0) - reservedCanvasWidth + return Math.max(available, 400) }, [workflowCanvasWidth, otherPanelWidth]) const updateNodePanelWidth = useCallback((width: number) => { // Ensure the width is within the min and max range - const newValue = Math.min(Math.max(width, 400), maxNodePanelWidth) + const newValue = Math.max(400, Math.min(width, maxNodePanelWidth)) localStorage.setItem('workflow-node-panel-width', `${newValue}`) setNodePanelWidth(newValue) }, [maxNodePanelWidth, setNodePanelWidth]) @@ -118,8 +119,13 @@ const BasePanel: FC = ({ useEffect(() => { if (!workflowCanvasWidth) return - if (workflowCanvasWidth - 400 <= nodePanelWidth + otherPanelWidth) - debounceUpdate(workflowCanvasWidth - 400 - otherPanelWidth) + + // If the total width of the three exceeds the canvas, shrink the node panel to the available range (at least 400px) + const total = nodePanelWidth + otherPanelWidth + reservedCanvasWidth + if (total > workflowCanvasWidth) { + const target = Math.max(workflowCanvasWidth - otherPanelWidth - reservedCanvasWidth, 400) + debounceUpdate(target) + } }, [nodePanelWidth, otherPanelWidth, workflowCanvasWidth, updateNodePanelWidth]) const { handleNodeSelect } = useNodesInteractions() diff --git a/web/app/components/workflow/nodes/_base/node.tsx b/web/app/components/workflow/nodes/_base/node.tsx index 88771d098e..b969702dd7 100644 --- a/web/app/components/workflow/nodes/_base/node.tsx +++ b/web/app/components/workflow/nodes/_base/node.tsx @@ -32,6 +32,7 @@ import { import { useNodeIterationInteractions } from '../iteration/use-interactions' import { useNodeLoopInteractions } from '../loop/use-interactions' import type { IterationNodeType } from '../iteration/types' +import CopyID from '../tool/components/copy-id' import { NodeSourceHandle, NodeTargetHandle, @@ -321,6 +322,11 @@ const BaseNode: FC = ({
) } + {data.type === BlockEnum.Tool && ( +
+ +
+ )}
) diff --git a/web/app/components/workflow/nodes/agent/components/tool-icon.tsx b/web/app/components/workflow/nodes/agent/components/tool-icon.tsx index b94258855a..8616f34200 100644 --- a/web/app/components/workflow/nodes/agent/components/tool-icon.tsx +++ b/web/app/components/workflow/nodes/agent/components/tool-icon.tsx @@ -2,10 +2,11 @@ import Tooltip from '@/app/components/base/tooltip' import Indicator from '@/app/components/header/indicator' import classNames from '@/utils/classnames' import { memo, useMemo, useRef, useState } from 'react' -import { useAllBuiltInTools, useAllCustomTools, useAllWorkflowTools } from '@/service/use-tools' +import { useAllBuiltInTools, useAllCustomTools, useAllMCPTools, useAllWorkflowTools } from '@/service/use-tools' import { getIconFromMarketPlace } from '@/utils/get-icon' import { useTranslation } from 'react-i18next' import { Group } from '@/app/components/base/icons/src/vender/other' +import AppIcon from '@/app/components/base/app-icon' type Status = 'not-installed' | 'not-authorized' | undefined @@ -19,19 +20,21 @@ export const ToolIcon = memo(({ providerName }: ToolIconProps) => { const { data: buildInTools } = useAllBuiltInTools() const { data: customTools } = useAllCustomTools() const { data: workflowTools } = useAllWorkflowTools() - const isDataReady = !!buildInTools && !!customTools && !!workflowTools + const { data: mcpTools } = useAllMCPTools() + const isDataReady = !!buildInTools && !!customTools && !!workflowTools && !!mcpTools const currentProvider = useMemo(() => { - const mergedTools = [...(buildInTools || []), ...(customTools || []), ...(workflowTools || [])] + const mergedTools = [...(buildInTools || []), ...(customTools || []), ...(workflowTools || []), ...(mcpTools || [])] return mergedTools.find((toolWithProvider) => { - return toolWithProvider.name === providerName + return toolWithProvider.name === providerName || toolWithProvider.id === providerName }) - }, [buildInTools, customTools, providerName, workflowTools]) + }, [buildInTools, customTools, providerName, workflowTools, mcpTools]) + const providerNameParts = providerName.split('/') const author = providerNameParts[0] const name = providerNameParts[1] const icon = useMemo(() => { if (!isDataReady) return '' - if (currentProvider) return currentProvider.icon as string + if (currentProvider) return currentProvider.icon const iconFromMarketPlace = getIconFromMarketPlace(`${author}/${name}`) return iconFromMarketPlace }, [author, currentProvider, name, isDataReady]) @@ -62,19 +65,32 @@ export const ToolIcon = memo(({ providerName }: ToolIconProps) => { )} ref={containerRef} > - {(!iconFetchError && isDataReady) - - ? tool icon setIconFetchError(true)} - /> - : - } + {(() => { + if (iconFetchError || !icon) + return + if (typeof icon === 'string') { + return tool icon setIconFetchError(true)} + /> + } + if (typeof icon === 'object') { + return + } + return + })()} {indicator && }
diff --git a/web/app/components/workflow/nodes/agent/default.ts b/web/app/components/workflow/nodes/agent/default.ts index d80def7bd2..4f68cfe87c 100644 --- a/web/app/components/workflow/nodes/agent/default.ts +++ b/web/app/components/workflow/nodes/agent/default.ts @@ -7,6 +7,7 @@ import { renderI18nObject } from '@/i18n' const nodeDefault: NodeDefault = { defaultValue: { + version: '2', }, getAvailablePrevNodes(isChatMode) { return isChatMode @@ -60,15 +61,28 @@ const nodeDefault: NodeDefault = { const schemas = toolValue.schemas || [] const userSettings = toolValue.settings const reasoningConfig = toolValue.parameters + const version = payload.version schemas.forEach((schema: any) => { if (schema?.required) { - if (schema.form === 'form' && !userSettings[schema.name]?.value) { + if (schema.form === 'form' && !version && !userSettings[schema.name]?.value) { return { isValid: false, errorMessage: t('workflow.errorMsg.toolParameterRequired', { field: renderI18nObject(param.label, language), param: renderI18nObject(schema.label, language) }), } } - if (schema.form === 'llm' && reasoningConfig[schema.name].auto === 0 && !userSettings[schema.name]?.value) { + if (schema.form === 'form' && version && !userSettings[schema.name]?.value.value) { + return { + isValid: false, + errorMessage: t('workflow.errorMsg.toolParameterRequired', { field: renderI18nObject(param.label, language), param: renderI18nObject(schema.label, language) }), + } + } + if (schema.form === 'llm' && !version && reasoningConfig[schema.name].auto === 0 && !reasoningConfig[schema.name]?.value) { + return { + isValid: false, + errorMessage: t('workflow.errorMsg.toolParameterRequired', { field: renderI18nObject(param.label, language), param: renderI18nObject(schema.label, language) }), + } + } + if (schema.form === 'llm' && version && reasoningConfig[schema.name].auto === 0 && !reasoningConfig[schema.name]?.value.value) { return { isValid: false, errorMessage: t('workflow.errorMsg.toolParameterRequired', { field: renderI18nObject(param.label, language), param: renderI18nObject(schema.label, language) }), diff --git a/web/app/components/workflow/nodes/agent/node.tsx b/web/app/components/workflow/nodes/agent/node.tsx index d2267fd00f..a2190317af 100644 --- a/web/app/components/workflow/nodes/agent/node.tsx +++ b/web/app/components/workflow/nodes/agent/node.tsx @@ -104,7 +104,7 @@ const AgentNode: FC> = (props) => { {t('workflow.nodes.agent.toolbox')} }>
- {tools.map(tool => )} + {tools.map((tool, i) => )}
}
diff --git a/web/app/components/workflow/nodes/agent/panel.tsx b/web/app/components/workflow/nodes/agent/panel.tsx index 391383031f..6741453944 100644 --- a/web/app/components/workflow/nodes/agent/panel.tsx +++ b/web/app/components/workflow/nodes/agent/panel.tsx @@ -38,11 +38,11 @@ const AgentPanel: FC> = (props) => { readOnly, outputSchema, handleMemoryChange, + canChooseMCPTool, } = useConfig(props.id, props.data) const { t } = useTranslation() const resetEditor = useStore(s => s.setControlPromptEditorRerenderKey) - return
> = (props) => { agent_strategy_label: inputs.agent_strategy_label!, agent_output_schema: inputs.output_schema, plugin_unique_identifier: inputs.plugin_unique_identifier!, + meta: inputs.meta, } : undefined} onStrategyChange={(strategy) => { setInputs({ @@ -65,6 +66,7 @@ const AgentPanel: FC> = (props) => { agent_strategy_label: strategy?.agent_strategy_label, output_schema: strategy!.agent_output_schema, plugin_unique_identifier: strategy!.plugin_unique_identifier, + meta: strategy?.meta, }) resetEditor(Date.now()) }} @@ -74,6 +76,7 @@ const AgentPanel: FC> = (props) => { nodeOutputVars={availableVars} availableNodes={availableNodesWithParent} nodeId={props.id} + canChooseMCPTool={canChooseMCPTool} />
diff --git a/web/app/components/workflow/nodes/agent/types.ts b/web/app/components/workflow/nodes/agent/types.ts index ca8bb5e71d..5a13a4a4f3 100644 --- a/web/app/components/workflow/nodes/agent/types.ts +++ b/web/app/components/workflow/nodes/agent/types.ts @@ -1,14 +1,17 @@ import type { CommonNodeType, Memory } from '@/app/components/workflow/types' import type { ToolVarInputs } from '../tool/types' +import type { PluginMeta } from '@/app/components/plugins/types' export type AgentNodeType = CommonNodeType & { agent_strategy_provider_name?: string agent_strategy_name?: string agent_strategy_label?: string agent_parameters?: ToolVarInputs + meta?: PluginMeta output_schema: Record plugin_unique_identifier?: string memory?: Memory + version?: string } export enum AgentFeature { diff --git a/web/app/components/workflow/nodes/agent/use-config.ts b/web/app/components/workflow/nodes/agent/use-config.ts index c3e07e4e60..50faf03040 100644 --- a/web/app/components/workflow/nodes/agent/use-config.ts +++ b/web/app/components/workflow/nodes/agent/use-config.ts @@ -6,13 +6,16 @@ import { useIsChatMode, useNodesReadOnly, } from '@/app/components/workflow/hooks' -import { useCallback, useMemo } from 'react' +import { useCallback, useEffect, useMemo } from 'react' import { type ToolVarInputs, VarType } from '../tool/types' import { useCheckInstalled, useFetchPluginsInMarketPlaceByIds } from '@/service/use-plugins' import type { Memory, Var } from '../../types' import { VarType as VarKindType } from '../../types' import useAvailableVarList from '../_base/hooks/use-available-var-list' import produce from 'immer' +import { isSupportMCP } from '@/utils/plugin-version-feature' +import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { generateAgentToolValue, toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' export type StrategyStatus = { plugin: { @@ -85,11 +88,12 @@ const useConfig = (id: string, payload: AgentNodeType) => { }) const formData = useMemo(() => { const paramNameList = (currentStrategy?.parameters || []).map(item => item.name) - return Object.fromEntries( + const res = Object.fromEntries( Object.entries(inputs.agent_parameters || {}).filter(([name]) => paramNameList.includes(name)).map(([key, value]) => { return [key, value.value] }), ) + return res }, [inputs.agent_parameters, currentStrategy?.parameters]) const onFormChange = (value: Record) => { const res: ToolVarInputs = {} @@ -105,6 +109,42 @@ const useConfig = (id: string, payload: AgentNodeType) => { }) } + const formattingToolData = (data: any) => { + const settingValues = generateAgentToolValue(data.settings, toolParametersToFormSchemas(data.schemas.filter((param: { form: string }) => param.form !== 'llm') as any)) + const paramValues = generateAgentToolValue(data.parameters, toolParametersToFormSchemas(data.schemas.filter((param: { form: string }) => param.form === 'llm') as any), true) + const res = produce(data, (draft: any) => { + draft.settings = settingValues + draft.parameters = paramValues + }) + return res + } + + const formattingLegacyData = () => { + if (inputs.version) + return inputs + const newData = produce(inputs, (draft) => { + const schemas = currentStrategy?.parameters || [] + Object.keys(draft.agent_parameters || {}).forEach((key) => { + const targetSchema = schemas.find(schema => schema.name === key) + if (targetSchema?.type === FormTypeEnum.toolSelector) + draft.agent_parameters![key].value = formattingToolData(draft.agent_parameters![key].value) + if (targetSchema?.type === FormTypeEnum.multiToolSelector) + draft.agent_parameters![key].value = draft.agent_parameters![key].value.map((tool: any) => formattingToolData(tool)) + }) + draft.version = '2' + }) + return newData + } + + // formatting legacy data + useEffect(() => { + if (!currentStrategy) + return + const newData = formattingLegacyData() + setInputs(newData) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentStrategy]) + // vars const filterMemoryPromptVar = useCallback((varPayload: Var) => { @@ -172,6 +212,7 @@ const useConfig = (id: string, payload: AgentNodeType) => { outputSchema, handleMemoryChange, isChatMode, + canChooseMCPTool: isSupportMCP(inputs.meta?.version), } } diff --git a/web/app/components/workflow/nodes/http/default.ts b/web/app/components/workflow/nodes/http/default.ts index 1bd584eeb9..3f9df0178d 100644 --- a/web/app/components/workflow/nodes/http/default.ts +++ b/web/app/components/workflow/nodes/http/default.ts @@ -22,6 +22,7 @@ const nodeDefault: NodeDefault = { type: BodyType.none, data: [], }, + ssl_verify: true, timeout: { max_connect_timeout: 0, max_read_timeout: 0, diff --git a/web/app/components/workflow/nodes/http/panel.tsx b/web/app/components/workflow/nodes/http/panel.tsx index 9a07c0ad61..b994910ea0 100644 --- a/web/app/components/workflow/nodes/http/panel.tsx +++ b/web/app/components/workflow/nodes/http/panel.tsx @@ -10,6 +10,7 @@ import type { HttpNodeType } from './types' import Timeout from './components/timeout' import CurlPanel from './components/curl-panel' import cn from '@/utils/classnames' +import Switch from '@/app/components/base/switch' import Field from '@/app/components/workflow/nodes/_base/components/field' import Split from '@/app/components/workflow/nodes/_base/components/split' import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars' @@ -47,6 +48,7 @@ const Panel: FC> = ({ showCurlPanel, hideCurlPanel, handleCurlImport, + handleSSLVerifyChange, } = useConfig(id, data) // To prevent prompt editor in body not update data. if (!isDataReady) @@ -124,6 +126,18 @@ const Panel: FC> = ({ onChange={setBody} /> + + }> +
{ setInputs(newInputs) }, [inputs, setInputs]) + const handleSSLVerifyChange = useCallback((checked: boolean) => { + const newInputs = produce(inputs, (draft: HttpNodeType) => { + draft.ssl_verify = checked + }) + setInputs(newInputs) + }, [inputs, setInputs]) + return { readOnly, isDataReady, @@ -164,6 +171,8 @@ const useConfig = (id: string, payload: HttpNodeType) => { toggleIsParamKeyValueEdit, // body setBody, + // ssl verify + handleSSLVerifyChange, // authorization isShowAuthorization, showAuthorization, diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-common-variable-selector.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-common-variable-selector.tsx index 00ba306d03..0d81d808db 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-common-variable-selector.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-common-variable-selector.tsx @@ -9,7 +9,7 @@ import type { VarType } from '@/app/components/workflow/types' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' type ConditionCommonVariableSelectorProps = { - variables?: { name: string; type: string }[] + variables?: { name: string; type: string; value: string }[] value?: string | number varType?: VarType onChange: (v: string) => void @@ -24,7 +24,7 @@ const ConditionCommonVariableSelector = ({ const { t } = useTranslation() const [open, setOpen] = useState(false) - const selected = variables.find(v => v.name === value) + const selected = variables.find(v => v.value === value) const handleChange = useCallback((v: string) => { onChange(v) setOpen(false) @@ -49,7 +49,7 @@ const ConditionCommonVariableSelector = ({ selected && (
- {selected.name} + {selected.value}
) } @@ -73,12 +73,12 @@ const ConditionCommonVariableSelector = ({ { variables.map(v => (
handleChange(v.name)} + onClick={() => handleChange(v.value)} > - {v.name} + {v.value}
)) } diff --git a/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/hooks.ts b/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/hooks.ts index 470a322b13..d11ad92241 100644 --- a/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/hooks.ts +++ b/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/hooks.ts @@ -6,6 +6,7 @@ import type { EditData } from './edit-card' import { ArrayType, type Field, Type } from '../../../types' import Toast from '@/app/components/base/toast' import { findPropertyWithPath } from '../../../utils' +import { noop } from 'lodash-es' type ChangeEventParams = { path: string[], @@ -19,7 +20,8 @@ type AddEventParams = { } export const useSchemaNodeOperations = (props: VisualEditorProps) => { - const { schema: jsonSchema, onChange } = props + const { schema: jsonSchema, onChange: doOnChange } = props + const onChange = doOnChange || noop const backupSchema = useVisualEditorStore(state => state.backupSchema) const setBackupSchema = useVisualEditorStore(state => state.setBackupSchema) const isAddingNewField = useVisualEditorStore(state => state.isAddingNewField) diff --git a/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/index.tsx b/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/index.tsx index 1df42532a6..d96f856bbb 100644 --- a/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/index.tsx +++ b/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/index.tsx @@ -2,24 +2,29 @@ import type { FC } from 'react' import type { SchemaRoot } from '../../../types' import SchemaNode from './schema-node' import { useSchemaNodeOperations } from './hooks' +import cn from '@/utils/classnames' export type VisualEditorProps = { + className?: string schema: SchemaRoot - onChange: (schema: SchemaRoot) => void + rootName?: string + readOnly?: boolean + onChange?: (schema: SchemaRoot) => void } const VisualEditor: FC = (props) => { - const { schema } = props + const { className, schema, readOnly } = props useSchemaNodeOperations(props) return ( -
+
) diff --git a/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/schema-node.tsx b/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/schema-node.tsx index 70a6b861ad..36671ab050 100644 --- a/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/schema-node.tsx +++ b/web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/schema-node.tsx @@ -19,6 +19,7 @@ type SchemaNodeProps = { path: string[] parentPath?: string[] depth: number + readOnly?: boolean } // Support 10 levels of indentation @@ -57,6 +58,7 @@ const SchemaNode: FC = ({ path, parentPath, depth, + readOnly, }) => { const [isExpanded, setIsExpanded] = useState(true) const hoveringProperty = useVisualEditorStore(state => state.hoveringProperty) @@ -77,11 +79,13 @@ const SchemaNode: FC = ({ } const handleMouseEnter = () => { + if(readOnly) return if (advancedEditing || isAddingNewField) return setHoveringPropertyDebounced(path.join('.')) } const handleMouseLeave = () => { + if(readOnly) return if (advancedEditing || isAddingNewField) return setHoveringPropertyDebounced(null) } @@ -91,7 +95,7 @@ const SchemaNode: FC = ({
{depth > 0 && hasChildren && (
= ({ )} { - depth === 0 && !isAddingNewField && ( + !readOnly && depth === 0 && !isAddingNewField && ( ) } diff --git a/web/app/components/workflow/nodes/question-classifier/components/class-item.tsx b/web/app/components/workflow/nodes/question-classifier/components/class-item.tsx index be4a6cb901..478ac925d6 100644 --- a/web/app/components/workflow/nodes/question-classifier/components/class-item.tsx +++ b/web/app/components/workflow/nodes/question-classifier/components/class-item.tsx @@ -11,6 +11,8 @@ import { uniqueId } from 'lodash-es' const i18nPrefix = 'workflow.nodes.questionClassifiers' type Props = { + className?: string + headerClassName?: string nodeId: string payload: Topic onChange: (payload: Topic) => void @@ -21,6 +23,8 @@ type Props = { } const ClassItem: FC = ({ + className, + headerClassName, nodeId, payload, onChange, @@ -49,6 +53,8 @@ const ClassItem: FC = ({ return ( void readonly?: boolean filterVar: (payload: Var, valueSelector: ValueSelector) => boolean + handleSortTopic?: (newTopics: (Topic & { id: string })[]) => void } const ClassList: FC = ({ @@ -25,6 +29,7 @@ const ClassList: FC = ({ onChange, readonly, filterVar, + handleSortTopic = noop, }) => { const { t } = useTranslation() const { handleEdgeDeleteByDeleteBranch } = useEdgesInteractions() @@ -55,22 +60,48 @@ const ClassList: FC = ({ } }, [list, onChange, handleEdgeDeleteByDeleteBranch, nodeId]) + const topicCount = list.length + const handleSideWidth = 3 // Todo Remove; edit topic name return ( -
+ ({ ...item }))} + setList={handleSortTopic} + handle='.handle' + ghostClass='bg-components-panel-bg' + animation={150} + disabled={readonly} + className='space-y-2' + > { list.map((item, index) => { + const canDrag = (() => { + if (readonly) + return false + + return topicCount >= 2 + })() return ( - +
+
+ +
+
) }) } @@ -81,7 +112,7 @@ const ClassList: FC = ({ /> )} -
+ ) } export default React.memo(ClassList) diff --git a/web/app/components/workflow/nodes/question-classifier/panel.tsx b/web/app/components/workflow/nodes/question-classifier/panel.tsx index 8cf9ec5f7c..8e27f5dceb 100644 --- a/web/app/components/workflow/nodes/question-classifier/panel.tsx +++ b/web/app/components/workflow/nodes/question-classifier/panel.tsx @@ -40,6 +40,7 @@ const Panel: FC> = ({ handleVisionResolutionChange, handleVisionResolutionEnabledChange, filterVar, + handleSortTopic, } = useConfig(id, data) const model = inputs.model @@ -99,6 +100,7 @@ const Panel: FC> = ({ onChange={handleTopicsChange} readonly={readOnly} filterVar={filterVar} + handleSortTopic={handleSortTopic} /> diff --git a/web/app/components/workflow/nodes/question-classifier/use-config.ts b/web/app/components/workflow/nodes/question-classifier/use-config.ts index 8eacf5b43f..a4acf5b7f6 100644 --- a/web/app/components/workflow/nodes/question-classifier/use-config.ts +++ b/web/app/components/workflow/nodes/question-classifier/use-config.ts @@ -9,13 +9,15 @@ import { import { useStore } from '../../store' import useAvailableVarList from '../_base/hooks/use-available-var-list' import useConfigVision from '../../hooks/use-config-vision' -import type { QuestionClassifierNodeType } from './types' +import type { QuestionClassifierNodeType, Topic } from './types' import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud' import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' import { checkHasQueryBlock } from '@/app/components/base/prompt-editor/constants' +import { useUpdateNodeInternals } from 'reactflow' const useConfig = (id: string, payload: QuestionClassifierNodeType) => { + const updateNodeInternals = useUpdateNodeInternals() const { nodesReadOnly: readOnly } = useNodesReadOnly() const isChatMode = useIsChatMode() const defaultConfig = useStore(s => s.nodesDefaultConfigs)[payload.type] @@ -166,6 +168,17 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { return varPayload.type === VarType.string }, []) + const handleSortTopic = useCallback((newTopics: (Topic & { id: string })[]) => { + const newInputs = produce(inputs, (draft) => { + draft.classes = newTopics.filter(Boolean).map(item => ({ + id: item.id, + name: item.name, + })) + }) + setInputs(newInputs) + updateNodeInternals(id) + }, [id, inputs, setInputs, updateNodeInternals]) + return { readOnly, inputs, @@ -185,6 +198,7 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { isVisionModel, handleVisionResolutionEnabledChange, handleVisionResolutionChange, + handleSortTopic, } } diff --git a/web/app/components/workflow/nodes/start/components/var-item.tsx b/web/app/components/workflow/nodes/start/components/var-item.tsx index 68dc141d75..029547542e 100644 --- a/web/app/components/workflow/nodes/start/components/var-item.tsx +++ b/web/app/components/workflow/nodes/start/components/var-item.tsx @@ -13,18 +13,22 @@ import { Edit03 } from '@/app/components/base/icons/src/vender/solid/general' import Badge from '@/app/components/base/badge' import ConfigVarModal from '@/app/components/app/configuration/config-var/config-modal' import { noop } from 'lodash-es' +import cn from '@/utils/classnames' type Props = { + className?: string readonly: boolean payload: InputVar onChange?: (item: InputVar, moreInfo?: MoreInfo) => void onRemove?: () => void rightContent?: React.JSX.Element varKeys?: string[] - showLegacyBadge?: boolean + showLegacyBadge?: boolean, + canDrag?: boolean, } const VarItem: FC = ({ + className, readonly, payload, onChange = noop, @@ -32,6 +36,7 @@ const VarItem: FC = ({ rightContent, varKeys = [], showLegacyBadge = false, + canDrag, }) => { const { t } = useTranslation() @@ -47,9 +52,9 @@ const VarItem: FC = ({ hideEditVarModal() }, [onChange, hideEditVarModal]) return ( -
+
- +
{payload.variable}
{payload.label && (<>
·
{payload.label as string}
diff --git a/web/app/components/workflow/nodes/start/components/var-list.tsx b/web/app/components/workflow/nodes/start/components/var-list.tsx index 7eccbec618..024b50a759 100644 --- a/web/app/components/workflow/nodes/start/components/var-list.tsx +++ b/web/app/components/workflow/nodes/start/components/var-list.tsx @@ -1,10 +1,15 @@ 'use client' import type { FC } from 'react' -import React, { useCallback } from 'react' +import React, { useCallback, useMemo } from 'react' import produce from 'immer' import { useTranslation } from 'react-i18next' import VarItem from './var-item' import { ChangeType, type InputVar, type MoreInfo } from '@/app/components/workflow/types' +import { v4 as uuid4 } from 'uuid' +import { ReactSortable } from 'react-sortablejs' +import { RiDraggable } from '@remixicon/react' +import cn from '@/utils/classnames' + type Props = { readonly: boolean list: InputVar[] @@ -44,6 +49,16 @@ const VarList: FC = ({ } }, [list, onChange]) + const listWithIds = useMemo(() => list.map((item) => { + const id = uuid4() + return { + id, + variable: { ...item }, + } + }), [list]) + + const varCount = list.length + if (list.length === 0) { return (
@@ -53,18 +68,39 @@ const VarList: FC = ({ } return ( -
- {list.map((item, index) => ( - item.variable)} - /> - ))} -
+ { onChange(list.map(item => item.variable)) }} + handle='.handle' + ghostClass='opacity-50' + animation={150} + > + {list.map((item, index) => { + const canDrag = (() => { + if (readonly) + return false + return varCount > 1 + })() + return ( +
+ item.variable)} + canDrag={canDrag} + /> + {canDrag &&
+ ) + })} + ) } export default React.memo(VarList) diff --git a/web/app/components/workflow/nodes/tool/components/copy-id.tsx b/web/app/components/workflow/nodes/tool/components/copy-id.tsx new file mode 100644 index 0000000000..3a633e1d2e --- /dev/null +++ b/web/app/components/workflow/nodes/tool/components/copy-id.tsx @@ -0,0 +1,51 @@ +'use client' +import React, { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { RiFileCopyLine } from '@remixicon/react' +import copy from 'copy-to-clipboard' +import { debounce } from 'lodash-es' +import Tooltip from '@/app/components/base/tooltip' + +type Props = { + content: string +} + +const prefixEmbedded = 'appOverview.overview.appInfo.embedded' + +const CopyFeedbackNew = ({ content }: Props) => { + const { t } = useTranslation() + const [isCopied, setIsCopied] = useState(false) + + const onClickCopy = debounce(() => { + copy(content) + setIsCopied(true) + }, 100) + + const onMouseLeave = debounce(() => { + setIsCopied(false) + }, 100) + + return ( +
e.stopPropagation()} onMouseLeave={onMouseLeave}> + +
+
{content}
+ +
+ +
+ ) +} + +export default CopyFeedbackNew diff --git a/web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/index.tsx b/web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/index.tsx new file mode 100644 index 0000000000..6680c8ebb6 --- /dev/null +++ b/web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/index.tsx @@ -0,0 +1,62 @@ +import { + memo, +} from 'react' +import { useTranslation } from 'react-i18next' +import PromptEditor from '@/app/components/base/prompt-editor' +import Placeholder from './placeholder' +import type { + Node, + NodeOutPutVar, +} from '@/app/components/workflow/types' +import { BlockEnum } from '@/app/components/workflow/types' +import cn from '@/utils/classnames' + +type MixedVariableTextInputProps = { + readOnly?: boolean + nodesOutputVars?: NodeOutPutVar[] + availableNodes?: Node[] + value?: string + onChange?: (text: string) => void +} +const MixedVariableTextInput = ({ + readOnly = false, + nodesOutputVars, + availableNodes = [], + value = '', + onChange, +}: MixedVariableTextInputProps) => { + const { t } = useTranslation() + return ( + { + acc[node.id] = { + title: node.data.title, + type: node.data.type, + } + if (node.data.type === BlockEnum.Start) { + acc.sys = { + title: t('workflow.blocks.start'), + type: BlockEnum.Start, + } + } + return acc + }, {} as any), + }} + placeholder={} + onChange={onChange} + /> + ) +} + +export default memo(MixedVariableTextInput) diff --git a/web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/placeholder.tsx b/web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/placeholder.tsx new file mode 100644 index 0000000000..3337d6ae66 --- /dev/null +++ b/web/app/components/workflow/nodes/tool/components/mixed-variable-text-input/placeholder.tsx @@ -0,0 +1,51 @@ +import { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' +import { FOCUS_COMMAND } from 'lexical' +import { $insertNodes } from 'lexical' +import { CustomTextNode } from '@/app/components/base/prompt-editor/plugins/custom-text/node' +import Badge from '@/app/components/base/badge' + +const Placeholder = () => { + const { t } = useTranslation() + const [editor] = useLexicalComposerContext() + + const handleInsert = useCallback((text: string) => { + editor.update(() => { + const textNode = new CustomTextNode(text) + $insertNodes([textNode]) + }) + editor.dispatchCommand(FOCUS_COMMAND, undefined as any) + }, [editor]) + + return ( +
{ + e.stopPropagation() + handleInsert('') + }} + > +
+ {t('workflow.nodes.tool.insertPlaceholder1')} +
/
+
{ + e.stopPropagation() + handleInsert('/') + })} + > + {t('workflow.nodes.tool.insertPlaceholder2')} +
+
+ +
+ ) +} + +export default Placeholder diff --git a/web/app/components/workflow/nodes/tool/components/tool-form/index.tsx b/web/app/components/workflow/nodes/tool/components/tool-form/index.tsx new file mode 100644 index 0000000000..a867797473 --- /dev/null +++ b/web/app/components/workflow/nodes/tool/components/tool-form/index.tsx @@ -0,0 +1,51 @@ +'use client' +import type { FC } from 'react' +import type { ToolVarInputs } from '../../types' +import type { CredentialFormSchema } from '@/app/components/header/account-setting/model-provider-page/declarations' +import ToolFormItem from './item' +import type { ToolWithProvider } from '@/app/components/workflow/types' +import type { Tool } from '@/app/components/tools/types' + +type Props = { + readOnly: boolean + nodeId: string + schema: CredentialFormSchema[] + value: ToolVarInputs + onChange: (value: ToolVarInputs) => void + onOpen?: (index: number) => void + inPanel?: boolean + currentTool?: Tool + currentProvider?: ToolWithProvider +} + +const ToolForm: FC = ({ + readOnly, + nodeId, + schema, + value, + onChange, + inPanel, + currentTool, + currentProvider, +}) => { + return ( +
+ { + schema.map((schema, index) => ( + + )) + } +
+ ) +} +export default ToolForm diff --git a/web/app/components/workflow/nodes/tool/components/tool-form/item.tsx b/web/app/components/workflow/nodes/tool/components/tool-form/item.tsx new file mode 100644 index 0000000000..11de42fe56 --- /dev/null +++ b/web/app/components/workflow/nodes/tool/components/tool-form/item.tsx @@ -0,0 +1,105 @@ +'use client' +import type { FC } from 'react' +import { + RiBracesLine, +} from '@remixicon/react' +import type { ToolVarInputs } from '../../types' +import type { CredentialFormSchema } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks' +import Button from '@/app/components/base/button' +import Tooltip from '@/app/components/base/tooltip' +import FormInputItem from '@/app/components/workflow/nodes/_base/components/form-input-item' +import { useBoolean } from 'ahooks' +import SchemaModal from '@/app/components/plugins/plugin-detail-panel/tool-selector/schema-modal' +import type { ToolWithProvider } from '@/app/components/workflow/types' +import type { Tool } from '@/app/components/tools/types' + +type Props = { + readOnly: boolean + nodeId: string + schema: CredentialFormSchema + value: ToolVarInputs + onChange: (value: ToolVarInputs) => void + inPanel?: boolean + currentTool?: Tool + currentProvider?: ToolWithProvider +} + +const ToolFormItem: FC = ({ + readOnly, + nodeId, + schema, + value, + onChange, + inPanel, + currentTool, + currentProvider, +}) => { + const language = useLanguage() + const { name, label, type, required, tooltip, input_schema } = schema + const showSchemaButton = type === FormTypeEnum.object || type === FormTypeEnum.array + const showDescription = type === FormTypeEnum.textInput || type === FormTypeEnum.secretInput + const [isShowSchema, { + setTrue: showSchema, + setFalse: hideSchema, + }] = useBoolean(false) + return ( +
+
+
+
{label[language] || label.en_US}
+ {required && ( +
*
+ )} + {!showDescription && tooltip && ( + + {tooltip[language] || tooltip.en_US} +
} + triggerClassName='ml-1 w-4 h-4' + asChild={false} + /> + )} + {showSchemaButton && ( + <> +
·
+ + + )} +
+ {showDescription && tooltip && ( +
{tooltip[language] || tooltip.en_US}
+ )} +
+ + + {isShowSchema && ( + + )} +
+ ) +} +export default ToolFormItem diff --git a/web/app/components/workflow/nodes/tool/default.ts b/web/app/components/workflow/nodes/tool/default.ts index f245929684..1fdb9eed2d 100644 --- a/web/app/components/workflow/nodes/tool/default.ts +++ b/web/app/components/workflow/nodes/tool/default.ts @@ -10,6 +10,7 @@ const nodeDefault: NodeDefault = { defaultValue: { tool_parameters: {}, tool_configurations: {}, + version: '2', }, getAvailablePrevNodes(isChatMode: boolean) { const nodes = isChatMode @@ -55,6 +56,8 @@ const nodeDefault: NodeDefault = { const value = payload.tool_configurations[field.variable] if (!errorMessages && (value === undefined || value === null || value === '')) errorMessages = t(`${i18nPrefix}.fieldRequired`, { field: field.label[language] }) + if (!errorMessages && typeof value === 'object' && !!value.type && (value.value === undefined || value.value === null || value.value === '' || (Array.isArray(value.value) && value.value.length === 0))) + errorMessages = t(`${i18nPrefix}.fieldRequired`, { field: field.label[language] }) }) } diff --git a/web/app/components/workflow/nodes/tool/node.tsx b/web/app/components/workflow/nodes/tool/node.tsx index f3cb4d9fae..e15ddcaaaa 100644 --- a/web/app/components/workflow/nodes/tool/node.tsx +++ b/web/app/components/workflow/nodes/tool/node.tsx @@ -21,14 +21,14 @@ const Node: FC> = ({
{key}
- {typeof tool_configurations[key] === 'string' && ( + {typeof tool_configurations[key].value === 'string' && (
- {paramSchemas?.find(i => i.name === key)?.type === FormTypeEnum.secretInput ? '********' : tool_configurations[key]} + {paramSchemas?.find(i => i.name === key)?.type === FormTypeEnum.secretInput ? '********' : tool_configurations[key].value}
)} - {typeof tool_configurations[key] === 'number' && ( + {typeof tool_configurations[key].value === 'number' && (
- {tool_configurations[key]} + {tool_configurations[key].value}
)} {typeof tool_configurations[key] !== 'string' && tool_configurations[key]?.type === FormTypeEnum.modelSelector && ( @@ -36,11 +36,6 @@ const Node: FC> = ({ {tool_configurations[key].model}
)} - {/* {typeof tool_configurations[key] !== 'string' && tool_configurations[key]?.type === FormTypeEnum.appSelector && ( -
- {tool_configurations[key].app_id} -
- )} */}
))} diff --git a/web/app/components/workflow/nodes/tool/panel.tsx b/web/app/components/workflow/nodes/tool/panel.tsx index 038159870e..936f730a46 100644 --- a/web/app/components/workflow/nodes/tool/panel.tsx +++ b/web/app/components/workflow/nodes/tool/panel.tsx @@ -4,11 +4,10 @@ import { useTranslation } from 'react-i18next' import Split from '../_base/components/split' import type { ToolNodeType } from './types' import useConfig from './use-config' -import InputVarList from './components/input-var-list' +import ToolForm from './components/tool-form' import Button from '@/app/components/base/button' import Field from '@/app/components/workflow/nodes/_base/components/field' import type { NodePanelProps } from '@/app/components/workflow/types' -import Form from '@/app/components/header/account-setting/model-provider-page/model-modal/Form' import ConfigCredential from '@/app/components/tools/setting/build-in/config-credentials' import Loading from '@/app/components/base/loading' import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars' @@ -28,8 +27,6 @@ const Panel: FC> = ({ inputs, toolInputVarSchema, setInputVar, - handleOnVarOpen, - filterVar, toolSettingSchema, toolSettingValue, setToolSettingValue, @@ -45,6 +42,8 @@ const Panel: FC> = ({ currTool, } = useConfig(id, data) + const [collapsed, setCollapsed] = React.useState(false) + if (isLoading) { return
@@ -66,21 +65,19 @@ const Panel: FC> = ({
)} - {!isShowAuthBtn && <> -
+ {!isShowAuthBtn && ( +
{toolInputVarSchema.length > 0 && ( - @@ -88,24 +85,29 @@ const Panel: FC> = ({ )} {toolInputVarSchema.length > 0 && toolSettingSchema.length > 0 && ( - + )} - + {toolSettingSchema.length > 0 && ( + <> + + + + + + )}
- } + )} {showSetAuth && ( output_schema: Record paramSchemas?: Record[] + version?: string } diff --git a/web/app/components/workflow/nodes/tool/use-config.ts b/web/app/components/workflow/nodes/tool/use-config.ts index b83ae8a07f..ea8d0e21ca 100644 --- a/web/app/components/workflow/nodes/tool/use-config.ts +++ b/web/app/components/workflow/nodes/tool/use-config.ts @@ -8,10 +8,12 @@ import { useLanguage } from '@/app/components/header/account-setting/model-provi import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud' import { CollectionType } from '@/app/components/tools/types' import { updateBuiltInToolCredential } from '@/service/tools' -import { addDefaultValue, toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' +import { + getConfiguredValue, + toolParametersToFormSchemas, +} from '@/app/components/tools/utils/to-form-schema' import Toast from '@/app/components/base/toast' -import { VarType as VarVarType } from '@/app/components/workflow/types' -import type { InputVar, Var } from '@/app/components/workflow/types' +import type { InputVar } from '@/app/components/workflow/types' import { useFetchToolsData, useNodesReadOnly, @@ -26,17 +28,18 @@ const useConfig = (id: string, payload: ToolNodeType) => { const language = useLanguage() const { inputs, setInputs: doSetInputs } = useNodeCrud(id, payload) /* - * tool_configurations: tool setting, not dynamic setting - * tool_parameters: tool dynamic setting(by user) + * tool_configurations: tool setting, not dynamic setting (form type = form) + * tool_parameters: tool dynamic setting(form type = llm) * output_schema: tool dynamic output */ - const { provider_id, provider_type, tool_name, tool_configurations, output_schema } = inputs + const { provider_id, provider_type, tool_name, tool_configurations, output_schema, tool_parameters } = inputs const isBuiltIn = provider_type === CollectionType.builtIn const buildInTools = useStore(s => s.buildInTools) const customTools = useStore(s => s.customTools) const workflowTools = useStore(s => s.workflowTools) + const mcpTools = useStore(s => s.mcpTools) - const currentTools = (() => { + const currentTools = useMemo(() => { switch (provider_type) { case CollectionType.builtIn: return buildInTools @@ -44,10 +47,12 @@ const useConfig = (id: string, payload: ToolNodeType) => { return customTools case CollectionType.workflow: return workflowTools + case CollectionType.mcp: + return mcpTools default: return [] } - })() + }, [buildInTools, customTools, mcpTools, provider_type, workflowTools]) const currCollection = currentTools.find(item => canFindTool(item.id, provider_id)) // Auth @@ -91,10 +96,10 @@ const useConfig = (id: string, payload: ToolNodeType) => { const value = newConfig[key] if (schema?.type === 'boolean') { if (typeof value === 'string') - newConfig[key] = Number.parseInt(value, 10) + newConfig[key] = value === 'true' || value === '1' - if (typeof value === 'boolean') - newConfig[key] = value ? 1 : 0 + if (typeof value === 'number') + newConfig[key] = value === 1 } if (schema?.type === 'number-input') { @@ -107,12 +112,11 @@ const useConfig = (id: string, payload: ToolNodeType) => { doSetInputs(newInputs) }, [doSetInputs, formSchemas, hasShouldTransferTypeSettingInput]) const [notSetDefaultValue, setNotSetDefaultValue] = useState(false) - const toolSettingValue = (() => { + const toolSettingValue = useMemo(() => { if (notSetDefaultValue) return tool_configurations - - return addDefaultValue(tool_configurations, toolSettingSchema) - })() + return getConfiguredValue(tool_configurations, toolSettingSchema) + }, [notSetDefaultValue, toolSettingSchema, tool_configurations]) const setToolSettingValue = useCallback((value: Record) => { setNotSetDefaultValue(true) setInputs({ @@ -121,16 +125,20 @@ const useConfig = (id: string, payload: ToolNodeType) => { }) }, [inputs, setInputs]) - useEffect(() => { - if (!currTool) - return + const formattingParameters = () => { const inputsWithDefaultValue = produce(inputs, (draft) => { if (!draft.tool_configurations || Object.keys(draft.tool_configurations).length === 0) - draft.tool_configurations = addDefaultValue(tool_configurations, toolSettingSchema) - - if (!draft.tool_parameters) - draft.tool_parameters = {} + draft.tool_configurations = getConfiguredValue(tool_configurations, toolSettingSchema) + if (!draft.tool_parameters || Object.keys(draft.tool_parameters).length === 0) + draft.tool_parameters = getConfiguredValue(tool_parameters, toolInputVarSchema) }) + return inputsWithDefaultValue + } + + useEffect(() => { + if (!currTool) + return + const inputsWithDefaultValue = formattingParameters() setInputs(inputsWithDefaultValue) // eslint-disable-next-line react-hooks/exhaustive-deps }, [currTool]) @@ -143,19 +151,6 @@ const useConfig = (id: string, payload: ToolNodeType) => { }) }, [inputs, setInputs]) - const [currVarIndex, setCurrVarIndex] = useState(-1) - const currVarType = toolInputVarSchema[currVarIndex]?._type - const handleOnVarOpen = useCallback((index: number) => { - setCurrVarIndex(index) - }, []) - - const filterVar = useCallback((varPayload: Var) => { - if (currVarType) - return varPayload.type === currVarType - - return varPayload.type !== VarVarType.arrayFile - }, [currVarType]) - const isLoading = currTool && (isBuiltIn ? !currCollection : false) const getMoreDataForCheckValid = () => { @@ -220,8 +215,6 @@ const useConfig = (id: string, payload: ToolNodeType) => { setToolSettingValue, toolInputVarSchema, setInputVar, - handleOnVarOpen, - filterVar, currCollection, isShowAuthBtn, showSetAuth, diff --git a/web/app/components/workflow/nodes/tool/use-single-run-form-params.ts b/web/app/components/workflow/nodes/tool/use-single-run-form-params.ts index 295cf02639..6fc79beebe 100644 --- a/web/app/components/workflow/nodes/tool/use-single-run-form-params.ts +++ b/web/app/components/workflow/nodes/tool/use-single-run-form-params.ts @@ -34,7 +34,12 @@ const useSingleRunFormParams = ({ const hadVarParams = Object.keys(inputs.tool_parameters) .filter(key => inputs.tool_parameters[key].type !== VarType.constant) .map(k => inputs.tool_parameters[k]) - const varInputs = getInputVars(hadVarParams.map((p) => { + + const hadVarSettings = Object.keys(inputs.tool_configurations) + .filter(key => typeof inputs.tool_configurations[key] === 'object' && inputs.tool_configurations[key].type && inputs.tool_configurations[key].type !== VarType.constant) + .map(k => inputs.tool_configurations[k]) + + const varInputs = getInputVars([...hadVarParams, ...hadVarSettings].map((p) => { if (p.type === VarType.variable) { // handle the old wrong value not crash the page if (!(p.value as any).join) @@ -55,8 +60,11 @@ const useSingleRunFormParams = ({ const res = produce(inputVarValues, (draft) => { Object.keys(inputs.tool_parameters).forEach((key: string) => { const { type, value } = inputs.tool_parameters[key] - if (type === VarType.constant && (value === undefined || value === null)) + if (type === VarType.constant && (value === undefined || value === null)) { + if(!draft.tool_parameters || !draft.tool_parameters[key]) + return draft[key] = value + } }) }) return res diff --git a/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx b/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx index 347c83c155..869317ca6a 100644 --- a/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx +++ b/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx @@ -80,7 +80,7 @@ const ChatVariableModal = ({ const [objectValue, setObjectValue] = React.useState([DEFAULT_OBJECT_VALUE]) const [editorContent, setEditorContent] = React.useState() const [editInJSON, setEditInJSON] = React.useState(false) - const [des, setDes] = React.useState('') + const [description, setDescription] = React.useState('') const editorMinHeight = useMemo(() => { if (type === ChatVarType.ArrayObject) @@ -237,7 +237,7 @@ const ChatVariableModal = ({ name, value_type: type, value: formatValue(value), - description: des, + description, }) onClose() } @@ -247,7 +247,7 @@ const ChatVariableModal = ({ setName(chatVar.name) setType(chatVar.value_type) setValue(chatVar.value) - setDes(chatVar.description) + setDescription(chatVar.description) setObjectValue(getObjectValue()) if (chatVar.value_type === ChatVarType.ArrayObject) { setEditorContent(JSON.stringify(chatVar.value)) @@ -385,9 +385,9 @@ const ChatVariableModal = ({