doc: add debugging steps in api/README.md

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

.devcontainer/post_create_command.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-npm add -g pnpm@10.13.1
+npm add -g pnpm@10.11.1
 cd web && pnpm install
 pipx install uv
 
@@ -12,4 +12,3 @@ echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f do
 echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env down"' >> ~/.bashrc
 
 source /home/vscode/.bashrc
-

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -8,15 +8,13 @@ body:
       label: Self Checks
       description: "To make sure we get to you in time, please check the following :)"
       options:
-        - label: I have read the [Contributing Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) and [Language Policy](https://github.com/langgenius/dify/issues/1542).
-          required: true
         - label: This is only for bug report, if you would like to ask a question, please head to [Discussions](https://github.com/langgenius/dify/discussions/categories/general).
           required: true
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to submit this report, otherwise it will be closed.
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
-        - label: 【中文用户 & Non English User】请使用英语提交，否则会被关闭 ：）
+        - label: "[FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:)"
           required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
@@ -44,22 +42,20 @@ body:
     attributes:
       label: Steps to reproduce
       description: We highly suggest including screenshots and a bug report log. Please use the right markdown syntax for code blocks.
-      placeholder: Having detailed steps helps us reproduce the bug. If you have logs, please use fenced code blocks (triple backticks ```) to format them.
+      placeholder: Having detailed steps helps us reproduce the bug.
     validations:
       required: true
 
   - type: textarea
     attributes:
       label: ✔️ Expected Behavior
-      description: Describe what you expected to happen.
+      placeholder: What were you expecting?
-      placeholder: What were you expecting? Please do not copy and paste the steps to reproduce here.
     validations:
-      required: true
+      required: false
 
   - type: textarea
     attributes:
       label: ❌ Actual Behavior
-      description: Describe what actually happened.
+      placeholder: What happened instead?
-      placeholder: What happened instead? Please do not copy and paste the steps to reproduce here.
     validations:
       required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +1,5 @@
 blank_issues_enabled: false
 contact_links:
-  - name: "\U0001F4A1 Model Providers & Plugins"
-    url: "https://github.com/langgenius/dify-official-plugins/issues/new/choose"
-    about: Report issues with official plugins or model providers, you will need to provide the plugin version and other relevant details.
-  - name: "\U0001F4AC Documentation Issues"
-    url: "https://github.com/langgenius/dify-docs/issues/new"
-    about: Report issues with the documentation, such as typos, outdated information, or missing content. Please provide the specific section and details of the issue.
   - name: "\U0001F4E7 Discussions"
     url: https://github.com/langgenius/dify/discussions/categories/general
-    about: General discussions and seek help from the community
+    about: General discussions and request help from the community

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -0,0 +1,24 @@
+name: "📚 Documentation Issue"
+description: Report issues in our documentation
+labels:
+  - documentation
+body:
+  - type: checkboxes
+    attributes:
+      label: Self Checks
+      description: "To make sure we get to you in time, please check the following :)"
+      options:
+        - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
+          required: true
+        - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "[FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:)"
+          required: true
+        - label: "Please do not modify this template :) and fill in all the required fields."
+          required: true
+  - type: textarea
+    attributes:
+      label: Provide a description of requested docs changes
+      placeholder: Briefly describe which document needs to be corrected and why.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -8,11 +8,11 @@ body:
       label: Self Checks
       description: "To make sure we get to you in time, please check the following :)"
       options:
-        - label: I have read the [Contributing Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) and [Language Policy](https://github.com/langgenius/dify/issues/1542).
-          required: true
         - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
           required: true
-        - label: I confirm that I am using English to submit this report, otherwise it will be closed.
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "[FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:)"
           required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -0,0 +1,55 @@
+name: "🌐 Localization/Translation issue"
+description: Report incorrect translations. [please use English :)]
+labels:
+  - translation
+body:
+  - type: checkboxes
+    attributes:
+      label: Self Checks
+      description: "To make sure we get to you in time, please check the following :)"
+      options:
+        - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
+          required: true
+        - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
+          required: true
+        - label: "[FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:)"
+          required: true
+        - label: "Please do not modify this template :) and fill in all the required fields."
+          required: true
+  - type: input
+    attributes:
+      label: Dify version
+      description: Hover over system tray icon or look at Settings
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Utility with translation issue
+      placeholder: Some area
+      description: Please input here the utility with the translation issue
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: 🌐 Language affected
+      placeholder: "German"
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: ❌ Actual phrase(s)
+      placeholder: What is there? Please include a screenshot as that is extremely helpful.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: ✔️ Expected phrase(s)
+      placeholder: What was expected?
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: ℹ Why is the current translation wrong
+      placeholder: Why do you feel this is incorrect?
+    validations:
+      required: true

.github/workflows/autofix.yml

@@ -1,27 +0,0 @@
-name: autofix.ci
-on:
-  workflow_call:
-  pull_request:
-  push:
-    branches: [ "main" ]
-permissions:
-  contents: read
-
-jobs:
-  autofix:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      # Use uv to ensure we have the same ruff version in CI and locally.
-      - uses: astral-sh/setup-uv@7edac99f961f18b581bbd960d59d049f04c0002f 
-      - run: |
-          cd api
-          uv sync --dev
-          # Fix lint errors
-          uv run ruff check --fix-only .
-          # Format code
-          uv run ruff format .
-
-      - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27
-

.github/workflows/style.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v45
         with:
           files: |
             api/**
@@ -75,7 +75,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v45
         with:
           files: web/**
 
@@ -113,7 +113,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v45
         with:
           files: |
             docker/generate_docker_compose
@@ -144,7 +144,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v45
         with:
           files: |
             **.sh
@@ -152,15 +152,13 @@ jobs:
             **.yml
             **Dockerfile
             dev/**
-            .editorconfig
 
       - name: Super-linter
-        uses: super-linter/super-linter/slim@v8
+        uses: super-linter/super-linter/slim@v7
         if: steps.changed-files.outputs.any_changed == 'true'
         env:
           BASH_SEVERITY: warning
-          DEFAULT_BRANCH: origin/main
+          DEFAULT_BRANCH: main
-          EDITORCONFIG_FILE_NAME: editorconfig-checker.json
           FILTER_REGEX_INCLUDE: pnpm-lock.yaml
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           IGNORE_GENERATED_FILES: true
@@ -170,6 +168,16 @@ jobs:
           # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck
           # VALIDATE_GITHUB_ACTIONS: true
           VALIDATE_DOCKERFILE_HADOLINT: true
-          VALIDATE_EDITORCONFIG: true
           VALIDATE_XML: true
           VALIDATE_YAML: true
+
+      - name: EditorConfig checks
+        uses: super-linter/super-linter/slim@v7
+        env:
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IGNORE_GENERATED_FILES: true
+          IGNORE_GITIGNORED_FILES: true
+          # EditorConfig validation
+          VALIDATE_EDITORCONFIG: true
+          EDITORCONFIG_FILE_NAME: editorconfig-checker.json

.github/workflows/web-tests.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v45
         with:
           files: web/**
 

README.md

@@ -1,7 +1,7 @@
 ![cover-v5-optimized](./images/GitHub_README_if.png)
 
 <p align="center">
-  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">Introducing Dify Workflow File Upload: Recreate Google NotebookLM Podcast111</a>
+  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">Introducing Dify Workflow File Upload: Recreate Google NotebookLM Podcast</a>
 </p>
 
 <p align="center">
@@ -54,7 +54,7 @@
   <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
-Dify is an open-source platform for developing LLM applications. Its intuitive interface combines agentic AI workflows, RAG pipelines, agent capabilities, model management, observability features, and more—allowing you to quickly move from prototype to production.
+Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features, and more, allowing you to quickly move from prototype to production.
 
 ## Quick start
 
@@ -65,7 +65,7 @@ Dify is an open-source platform for developing LLM applications. Its intuitive i
 
 </br>
 
-The easiest way to start the Dify server is through [Docker Compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
+The easiest way to start the Dify server is through [docker compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
 
 ```bash
 cd dify
@@ -205,7 +205,6 @@ If you'd like to configure a highly-available setup, there are community-contrib
 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Using Terraform for Deployment
 
@@ -262,8 +261,8 @@ At the same time, please consider supporting Dify by sharing it on social media
 
 ## Security disclosure
 
-To protect your privacy, please avoid posting security issues on GitHub. Instead, report issues to security@dify.ai, and our team will respond with detailed answer.
+To protect your privacy, please avoid posting security issues on GitHub. Instead, send your questions to security@dify.ai and we will provide you with a more detailed answer.
 
 ## License
 
-This repository is licensed under the [Dify Open Source License](LICENSE), based on Apache 2.0 with additional conditions.
+This repository is available under the [Dify Open Source License](LICENSE), which is essentially Apache 2.0 with a few additional restrictions.

README_AR.md

@@ -188,7 +188,6 @@ docker compose up -d
 - [رسم بياني Helm من قبل @magicsong](https://github.com/magicsong/ai-charts)
 - [ملف YAML من قبل @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [ملف YAML من قبل @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 جديد! ملفات YAML (تدعم Dify v1.6.0) بواسطة @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### استخدام Terraform للتوزيع
 

README_BN.md

@@ -204,8 +204,6 @@ GitHub-এ ডিফাইকে স্টার দিয়ে রাখুন
 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 নতুন! YAML ফাইলসমূহ (Dify v1.6.0 সমর্থিত) তৈরি করেছেন @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
-
 
 #### টেরাফর্ম ব্যবহার করে ডিপ্লয়
 

README_CN.md

@@ -194,9 +194,9 @@ docker compose up -d
 
 如果您需要自定义配置，请参考 [.env.example](docker/.env.example) 文件中的注释，并更新 `.env` 文件中对应的值。此外，您可能需要根据您的具体部署环境和需求对 `docker-compose.yaml` 文件本身进行调整，例如更改镜像版本、端口映射或卷挂载。完成任何更改后，请重新运行 `docker-compose up -d`。您可以在[此处](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用环境变量的完整列表。
 
-#### 使用 Helm Chart 或 Kubernetes 资源清单（YAML）部署
+#### 使用 Helm Chart 部署
 
-使用 [Helm Chart](https://helm.sh/) 版本或者 Kubernetes 资源清单（YAML），可以在 Kubernetes 上部署 Dify。
+使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
@@ -204,10 +204,6 @@ docker compose up -d
 - [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
 
-- [🚀 NEW! YAML 文件 (支持 Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
-
-
-
 #### 使用 Terraform 部署
 
 使用 [terraform](https://www.terraform.io/) 一键将 Dify 部署到云平台

README_DE.md

@@ -203,7 +203,6 @@ Falls Sie eine hochverfügbare Konfiguration einrichten möchten, gibt es von de
 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Terraform für die Bereitstellung verwenden
 

README_ES.md

@@ -203,7 +203,6 @@ Si desea configurar una configuración de alta disponibilidad, la comunidad prop
 - [Gráfico Helm por @magicsong](https://github.com/magicsong/ai-charts)
 - [Ficheros YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [Ficheros YAML por @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 ¡NUEVO! Archivos YAML (compatible con Dify v1.6.0) por @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Uso de Terraform para el despliegue
 

README_FR.md

@@ -201,7 +201,6 @@ Si vous souhaitez configurer une configuration haute disponibilité, la communau
 - [Helm Chart par @magicsong](https://github.com/magicsong/ai-charts)
 - [Fichier YAML par @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [Fichier YAML par @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NOUVEAU ! Fichiers YAML (compatible avec Dify v1.6.0) par @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Utilisation de Terraform pour le déploiement
 

README_JA.md

@@ -202,7 +202,6 @@ docker compose up -d
 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 新着！YAML ファイル（Dify v1.6.0 対応）by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Terraformを使用したデプロイ
 

README_KL.md

@@ -201,7 +201,6 @@ If you'd like to configure a highly-available setup, there are community-contrib
 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Terraform atorlugu pilersitsineq
 

README_KR.md

@@ -195,7 +195,6 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했
 - [Helm Chart by @magicsong](https://github.com/magicsong/ai-charts)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Terraform을 사용한 배포
 

README_PT.md

@@ -200,7 +200,6 @@ Se deseja configurar uma instalação de alta disponibilidade, há [Helm Charts]
 - [Helm Chart de @magicsong](https://github.com/magicsong/ai-charts)
 - [Arquivo YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [Arquivo YAML por @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NOVO! Arquivos YAML (Compatível com Dify v1.6.0) por @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Usando o Terraform para Implantação
 

README_SI.md

@@ -201,7 +201,6 @@ Star Dify on GitHub and be instantly notified of new releases.
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [YAML file by @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NEW! YAML files (Supports Dify v1.6.0) by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Uporaba Terraform za uvajanje
 

README_TR.md

@@ -194,7 +194,6 @@ Yüksek kullanılabilirliğe sahip bir kurulum yapılandırmak isterseniz, Dify'
 - [@BorisPolonsky tarafından Helm Chart](https://github.com/BorisPolonsky/dify-helm)
 - [@Winson-030 tarafından YAML dosyası](https://github.com/Winson-030/dify-kubernetes)
 - [@wyy-holding tarafından YAML dosyası](https://github.com/wyy-holding/dify-k8s)
-- [🚀 YENİ! YAML dosyaları (Dify v1.6.0 destekli) @Zhoneym tarafından](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Dağıtım için Terraform Kullanımı
 

README_TW.md

@@ -197,13 +197,12 @@ Dify 的所有功能都提供相應的 API，因此您可以輕鬆地將 Dify
 
 如果您需要自定義配置，請參考我們的 [.env.example](docker/.env.example) 文件中的註釋，並在您的 `.env` 文件中更新相應的值。此外，根據您特定的部署環境和需求，您可能需要調整 `docker-compose.yaml` 文件本身，例如更改映像版本、端口映射或卷掛載。進行任何更改後，請重新運行 `docker-compose up -d`。您可以在[這裡](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用環境變數的完整列表。
 
-如果您想配置高可用性設置，社區貢獻的 [Helm Charts](https://helm.sh/) 和 Kubernetes 資源清單（YAML）允許在 Kubernetes 上部署 Dify。
+如果您想配置高可用性設置，社區貢獻的 [Helm Charts](https://helm.sh/) 和 YAML 文件允許在 Kubernetes 上部署 Dify。
 
 - [由 @LeoQuote 提供的 Helm Chart](https://github.com/douban/charts/tree/master/charts/dify)
 - [由 @BorisPolonsky 提供的 Helm Chart](https://github.com/BorisPolonsky/dify-helm)
 - [由 @Winson-030 提供的 YAML 文件](https://github.com/Winson-030/dify-kubernetes)
 - [由 @wyy-holding 提供的 YAML 文件](https://github.com/wyy-holding/dify-k8s)
-- [🚀 NEW! YAML 檔案（支援 Dify v1.6.0）by @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 ### 使用 Terraform 進行部署
 

README_VI.md

@@ -196,7 +196,6 @@ Nếu bạn muốn cấu hình một cài đặt có độ sẵn sàng cao, có
 - [Helm Chart bởi @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [Tệp YAML bởi @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 - [Tệp YAML bởi @wyy-holding](https://github.com/wyy-holding/dify-k8s)
-- [🚀 MỚI! Tệp YAML (Hỗ trợ Dify v1.6.0) bởi @Zhoneym](https://github.com/Zhoneym/DifyAI-Kubernetes)
 
 #### Sử dụng Terraform để Triển khai
 

api/.env.example

@@ -5,22 +5,17 @@
 SECRET_KEY=
 
 # Console API base URL
-CONSOLE_API_URL=http://localhost:5001
+CONSOLE_API_URL=http://127.0.0.1:5001
-CONSOLE_WEB_URL=http://localhost:3000
+CONSOLE_WEB_URL=http://127.0.0.1:3000
 
 # Service API base URL
-SERVICE_API_URL=http://localhost:5001
+SERVICE_API_URL=http://127.0.0.1:5001
 
 # Web APP base URL
-APP_WEB_URL=http://localhost:3000
+APP_WEB_URL=http://127.0.0.1:3000
 
 # Files URL
-FILES_URL=http://localhost:5001
+FILES_URL=http://127.0.0.1:5001
-
-# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network.
-# Set this to the internal Docker service URL for proper plugin file access.
-# Example: INTERNAL_FILES_URL=http://api:5001
-INTERNAL_FILES_URL=http://127.0.0.1:5001
 
 # The time in seconds after the signature is rejected
 FILES_ACCESS_TIMEOUT=300
@@ -54,7 +49,7 @@ REDIS_CLUSTERS_PASSWORD=
 
 # celery configuration
 CELERY_BROKER_URL=redis://:difyai123456@localhost:${REDIS_PORT}/1
-CELERY_BACKEND=redis
+
 # PostgreSQL database configuration
 DB_USERNAME=postgres
 DB_PASSWORD=difyai123456
@@ -138,14 +133,12 @@ SUPABASE_API_KEY=your-access-key
 SUPABASE_URL=your-server-url
 
 # CORS configuration
-WEB_API_CORS_ALLOW_ORIGINS=http://localhost:3000,*
+WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
-CONSOLE_CORS_ALLOW_ORIGINS=http://localhost:3000,*
+CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
 # Vector database configuration
-# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`.
+# support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase, opengauss, tablestore, matrixone
 VECTOR_STORE=weaviate
-# Prefix used to create collection name in vector database
-VECTOR_INDEX_NAME_PREFIX=Vector_index
 
 # Weaviate configuration
 WEAVIATE_ENDPOINT=http://localhost:8080
@@ -451,19 +444,6 @@ MAX_VARIABLE_SIZE=204800
 # hybrid: Save new data to object storage, read from both object storage and RDBMS
 WORKFLOW_NODE_EXECUTION_STORAGE=rdbms
 
-# Repository configuration
-# Core workflow execution repository implementation
-CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository
-
-# Core workflow node execution repository implementation
-CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository
-
-# API workflow node execution repository implementation
-API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository
-
-# API workflow run repository implementation
-API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository
-
 # App configuration
 APP_MAX_EXECUTION_TIME=1200
 APP_MAX_ACTIVE_REQUESTS=0
@@ -507,8 +487,6 @@ ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
-CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
-OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
 
 CREATE_TIDB_SERVICE_JOB_ENABLED=false
 
@@ -519,8 +497,6 @@ LOGIN_LOCKOUT_DURATION=86400
 
 # Enable OpenTelemetry
 ENABLE_OTEL=false
-OTLP_TRACE_ENDPOINT=
-OTLP_METRIC_ENDPOINT=
 OTLP_BASE_ENDPOINT=http://localhost:4318
 OTLP_API_KEY=
 OTEL_EXPORTER_OTLP_PROTOCOL=

api/Dockerfile

@@ -47,8 +47,6 @@ RUN \
         curl nodejs libgmp-dev libmpfr-dev libmpc-dev \
         # For Security
         expat libldap-2.5-0 perl libsqlite3-0 zlib1g \
-        # install fonts to support the use of tools like pypdfium2
-        fonts-noto-cjk \
         # install a package to improve the accuracy of guessing mime type and file extension
         media-types \
         # install libmagic to support the use of python-magic guess MIMETYPE

api/commands.py

@@ -2,22 +2,19 @@ import base64
 import json
 import logging
 import secrets
-from typing import Any, Optional
+from typing import Optional
 
 import click
 from flask import current_app
-from pydantic import TypeAdapter
 from sqlalchemy import select
 from werkzeug.exceptions import NotFound
 
 from configs import dify_config
 from constants.languages import languages
-from core.plugin.entities.plugin import ToolProviderID
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.rag.models.document import Document
-from core.tools.utils.system_oauth_encryption import encrypt_system_oauth_params
 from events.app_event import app_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
@@ -30,7 +27,6 @@ from models.dataset import Dataset, DatasetCollectionBinding, DatasetMetadata, D
 from models.dataset import Document as DatasetDocument
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
 from models.provider import Provider, ProviderModel
-from models.tools import ToolOAuthSystemClient
 from services.account_service import AccountService, RegisterService, TenantService
 from services.clear_free_plan_tenant_expired_logs import ClearFreePlanTenantExpiredLogs
 from services.plugin.data_migration import PluginDataMigration
@@ -50,7 +46,7 @@ def reset_password(email, new_password, password_confirm):
         click.echo(click.style("Passwords do not match.", fg="red"))
         return
 
-    account = db.session.query(Account).where(Account.email == email).one_or_none()
+    account = db.session.query(Account).filter(Account.email == email).one_or_none()
 
     if not account:
         click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
@@ -89,7 +85,7 @@ def reset_email(email, new_email, email_confirm):
         click.echo(click.style("New emails do not match.", fg="red"))
         return
 
-    account = db.session.query(Account).where(Account.email == email).one_or_none()
+    account = db.session.query(Account).filter(Account.email == email).one_or_none()
 
     if not account:
         click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
@@ -136,8 +132,8 @@ def reset_encrypt_key_pair():
 
         tenant.encrypt_public_key = generate_key_pair(tenant.id)
 
-        db.session.query(Provider).where(Provider.provider_type == "custom", Provider.tenant_id == tenant.id).delete()
+        db.session.query(Provider).filter(Provider.provider_type == "custom", Provider.tenant_id == tenant.id).delete()
-        db.session.query(ProviderModel).where(ProviderModel.tenant_id == tenant.id).delete()
+        db.session.query(ProviderModel).filter(ProviderModel.tenant_id == tenant.id).delete()
         db.session.commit()
 
         click.echo(
@@ -172,7 +168,7 @@ def migrate_annotation_vector_database():
             per_page = 50
             apps = (
                 db.session.query(App)
-                .where(App.status == "normal")
+                .filter(App.status == "normal")
                 .order_by(App.created_at.desc())
                 .limit(per_page)
                 .offset((page - 1) * per_page)
@@ -192,7 +188,7 @@ def migrate_annotation_vector_database():
             try:
                 click.echo("Creating app annotation index: {}".format(app.id))
                 app_annotation_setting = (
-                    db.session.query(AppAnnotationSetting).where(AppAnnotationSetting.app_id == app.id).first()
+                    db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app.id).first()
                 )
 
                 if not app_annotation_setting:
@@ -202,13 +198,13 @@ def migrate_annotation_vector_database():
                 # get dataset_collection_binding info
                 dataset_collection_binding = (
                     db.session.query(DatasetCollectionBinding)
-                    .where(DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id)
+                    .filter(DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id)
                     .first()
                 )
                 if not dataset_collection_binding:
                     click.echo("App annotation collection binding not found: {}".format(app.id))
                     continue
-                annotations = db.session.query(MessageAnnotation).where(MessageAnnotation.app_id == app.id).all()
+                annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all()
                 dataset = Dataset(
                     id=app.id,
                     tenant_id=app.tenant_id,
@@ -305,7 +301,7 @@ def migrate_knowledge_vector_database():
     while True:
         try:
             stmt = (
-                select(Dataset).where(Dataset.indexing_technique == "high_quality").order_by(Dataset.created_at.desc())
+                select(Dataset).filter(Dataset.indexing_technique == "high_quality").order_by(Dataset.created_at.desc())
             )
 
             datasets = db.paginate(select=stmt, page=page, per_page=50, max_per_page=50, error_out=False)
@@ -332,7 +328,7 @@ def migrate_knowledge_vector_database():
                     if dataset.collection_binding_id:
                         dataset_collection_binding = (
                             db.session.query(DatasetCollectionBinding)
-                            .where(DatasetCollectionBinding.id == dataset.collection_binding_id)
+                            .filter(DatasetCollectionBinding.id == dataset.collection_binding_id)
                             .one_or_none()
                         )
                         if dataset_collection_binding:
@@ -367,7 +363,7 @@ def migrate_knowledge_vector_database():
 
                 dataset_documents = (
                     db.session.query(DatasetDocument)
-                    .where(
+                    .filter(
                         DatasetDocument.dataset_id == dataset.id,
                         DatasetDocument.indexing_status == "completed",
                         DatasetDocument.enabled == True,
@@ -381,7 +377,7 @@ def migrate_knowledge_vector_database():
                 for dataset_document in dataset_documents:
                     segments = (
                         db.session.query(DocumentSegment)
-                        .where(
+                        .filter(
                             DocumentSegment.document_id == dataset_document.id,
                             DocumentSegment.status == "completed",
                             DocumentSegment.enabled == True,
@@ -468,7 +464,7 @@ def convert_to_agent_apps():
                 app_id = str(i.id)
                 if app_id not in proceeded_app_ids:
                     proceeded_app_ids.append(app_id)
-                    app = db.session.query(App).where(App.id == app_id).first()
+                    app = db.session.query(App).filter(App.id == app_id).first()
                     if app is not None:
                         apps.append(app)
 
@@ -483,7 +479,7 @@ def convert_to_agent_apps():
                 db.session.commit()
 
                 # update conversation mode to agent
-                db.session.query(Conversation).where(Conversation.app_id == app.id).update(
+                db.session.query(Conversation).filter(Conversation.app_id == app.id).update(
                     {Conversation.mode: AppMode.AGENT_CHAT.value}
                 )
 
@@ -560,7 +556,7 @@ def old_metadata_migration():
         try:
             stmt = (
                 select(DatasetDocument)
-                .where(DatasetDocument.doc_metadata.is_not(None))
+                .filter(DatasetDocument.doc_metadata.is_not(None))
                 .order_by(DatasetDocument.created_at.desc())
             )
             documents = db.paginate(select=stmt, page=page, per_page=50, max_per_page=50, error_out=False)
@@ -578,7 +574,7 @@ def old_metadata_migration():
                     else:
                         dataset_metadata = (
                             db.session.query(DatasetMetadata)
-                            .where(DatasetMetadata.dataset_id == document.dataset_id, DatasetMetadata.name == key)
+                            .filter(DatasetMetadata.dataset_id == document.dataset_id, DatasetMetadata.name == key)
                             .first()
                         )
                         if not dataset_metadata:
@@ -602,7 +598,7 @@ def old_metadata_migration():
                         else:
                             dataset_metadata_binding = (
                                 db.session.query(DatasetMetadataBinding)  # type: ignore
-                                .where(
+                                .filter(
                                     DatasetMetadataBinding.dataset_id == document.dataset_id,
                                     DatasetMetadataBinding.document_id == document.id,
                                     DatasetMetadataBinding.metadata_id == dataset_metadata.id,
@@ -717,7 +713,7 @@ where sites.id is null limit 1000"""
                     continue
 
                 try:
-                    app = db.session.query(App).where(App.id == app_id).first()
+                    app = db.session.query(App).filter(App.id == app_id).first()
                     if not app:
                         print(f"App {app_id} not found")
                         continue
@@ -1159,49 +1155,3 @@ def remove_orphaned_files_on_storage(force: bool):
         click.echo(click.style(f"Removed {removed_files} orphaned files without errors.", fg="green"))
     else:
         click.echo(click.style(f"Removed {removed_files} orphaned files, with {error_files} errors.", fg="yellow"))
-
-
-@click.command("setup-system-tool-oauth-client", help="Setup system tool oauth client.")
-@click.option("--provider", prompt=True, help="Provider name")
-@click.option("--client-params", prompt=True, help="Client Params")
-def setup_system_tool_oauth_client(provider, client_params):
-    """
-    Setup system tool oauth client
-    """
-    provider_id = ToolProviderID(provider)
-    provider_name = provider_id.provider_name
-    plugin_id = provider_id.plugin_id
-
-    try:
-        # json validate
-        click.echo(click.style(f"Validating client params: {client_params}", fg="yellow"))
-        client_params_dict = TypeAdapter(dict[str, Any]).validate_json(client_params)
-        click.echo(click.style("Client params validated successfully.", fg="green"))
-
-        click.echo(click.style(f"Encrypting client params: {client_params}", fg="yellow"))
-        click.echo(click.style(f"Using SECRET_KEY: `{dify_config.SECRET_KEY}`", fg="yellow"))
-        oauth_client_params = encrypt_system_oauth_params(client_params_dict)
-        click.echo(click.style("Client params encrypted successfully.", fg="green"))
-    except Exception as e:
-        click.echo(click.style(f"Error parsing client params: {str(e)}", fg="red"))
-        return
-
-    deleted_count = (
-        db.session.query(ToolOAuthSystemClient)
-        .filter_by(
-            provider=provider_name,
-            plugin_id=plugin_id,
-        )
-        .delete()
-    )
-    if deleted_count > 0:
-        click.echo(click.style(f"Deleted {deleted_count} existing oauth client params.", fg="yellow"))
-
-    oauth_client = ToolOAuthSystemClient(
-        provider=provider_name,
-        plugin_id=plugin_id,
-        encrypted_oauth_params=oauth_client_params,
-    )
-    db.session.add(oauth_client)
-    db.session.commit()
-    click.echo(click.style(f"OAuth client params setup successfully. id: {oauth_client.id}", fg="green"))

api/configs/feature/__init__.py

@@ -31,15 +31,6 @@ class SecurityConfig(BaseSettings):
         description="Duration in minutes for which a password reset token remains valid",
         default=5,
     )
-    CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
-        description="Duration in minutes for which a change email token remains valid",
-        default=5,
-    )
-
-    OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
-        description="Duration in minutes for which a owner transfer token remains valid",
-        default=5,
-    )
 
     LOGIN_DISABLED: bool = Field(
         description="Whether to disable login checks",
@@ -246,13 +237,6 @@ class FileAccessConfig(BaseSettings):
         default="",
     )
 
-    INTERNAL_FILES_URL: str = Field(
-        description="Internal base URL for file access within Docker network,"
-        " used for plugin daemon and internal service communication."
-        " Falls back to FILES_URL if not specified.",
-        default="",
-    )
-
     FILES_ACCESS_TIMEOUT: int = Field(
         description="Expiration time in seconds for file access URLs",
         default=300,
@@ -546,33 +530,6 @@ class WorkflowNodeExecutionConfig(BaseSettings):
     )
 
 
-class RepositoryConfig(BaseSettings):
-    """
-    Configuration for repository implementations
-    """
-
-    CORE_WORKFLOW_EXECUTION_REPOSITORY: str = Field(
-        description="Repository implementation for WorkflowExecution. Specify as a module path",
-        default="core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository",
-    )
-
-    CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: str = Field(
-        description="Repository implementation for WorkflowNodeExecution. Specify as a module path",
-        default="core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository",
-    )
-
-    API_WORKFLOW_NODE_EXECUTION_REPOSITORY: str = Field(
-        description="Service-layer repository implementation for WorkflowNodeExecutionModel operations. "
-        "Specify as a module path",
-        default="repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository",
-    )
-
-    API_WORKFLOW_RUN_REPOSITORY: str = Field(
-        description="Service-layer repository implementation for WorkflowRun operations. Specify as a module path",
-        default="repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository",
-    )
-
-
 class AuthConfig(BaseSettings):
     """
     Configuration for authentication and OAuth
@@ -623,16 +580,6 @@ class AuthConfig(BaseSettings):
         default=86400,
     )
 
-    CHANGE_EMAIL_LOCKOUT_DURATION: PositiveInt = Field(
-        description="Time (in seconds) a user must wait before retrying change email after exceeding the rate limit.",
-        default=86400,
-    )
-
-    OWNER_TRANSFER_LOCKOUT_DURATION: PositiveInt = Field(
-        description="Time (in seconds) a user must wait before retrying owner transfer after exceeding the rate limit.",
-        default=86400,
-    )
-
 
 class ModerationConfig(BaseSettings):
     """
@@ -984,7 +931,6 @@ class FeatureConfig(
     MultiModalTransferConfig,
     PositionConfig,
     RagEtlConfig,
-    RepositoryConfig,
     SecurityConfig,
     ToolConfig,
     UpdateConfig,

api/configs/middleware/__init__.py

@@ -85,11 +85,6 @@ class VectorStoreConfig(BaseSettings):
         default=False,
     )
 
-    VECTOR_INDEX_NAME_PREFIX: Optional[str] = Field(
-        description="Prefix used to create collection name in vector database",
-        default="Vector_index",
-    )
-
 
 class KeywordStoreConfig(BaseSettings):
     KEYWORD_STORE: str = Field(
@@ -167,11 +162,6 @@ class DatabaseConfig(BaseSettings):
         default=3600,
     )
 
-    SQLALCHEMY_POOL_USE_LIFO: bool = Field(
-        description="If True, SQLAlchemy will use last-in-first-out way to retrieve connections from pool.",
-        default=False,
-    )
-
     SQLALCHEMY_POOL_PRE_PING: bool = Field(
         description="If True, enables connection pool pre-ping feature to check connections.",
         default=False,
@@ -209,14 +199,13 @@ class DatabaseConfig(BaseSettings):
             "pool_recycle": self.SQLALCHEMY_POOL_RECYCLE,
             "pool_pre_ping": self.SQLALCHEMY_POOL_PRE_PING,
             "connect_args": connect_args,
-            "pool_use_lifo": self.SQLALCHEMY_POOL_USE_LIFO,
         }
 
 
 class CeleryConfig(DatabaseConfig):
     CELERY_BACKEND: str = Field(
         description="Backend for Celery task results. Options: 'database', 'redis'.",
-        default="redis",
+        default="database",
     )
 
     CELERY_BROKER_URL: Optional[str] = Field(

api/configs/observability/otel/otel_config.py

@@ -12,16 +12,6 @@ class OTelConfig(BaseSettings):
         default=False,
     )
 
-    OTLP_TRACE_ENDPOINT: str = Field(
-        description="OTLP trace endpoint",
-        default="",
-    )
-
-    OTLP_METRIC_ENDPOINT: str = Field(
-        description="OTLP metric endpoint",
-        default="",
-    )
-
     OTLP_BASE_ENDPOINT: str = Field(
         description="OTLP base endpoint",
         default="http://localhost:4318",

api/constants/__init__.py

@@ -1,7 +1,6 @@
 from configs import dify_config
 
 HIDDEN_VALUE = "[__HIDDEN__]"
-UNKNOWN_VALUE = "[__UNKNOWN__]"
 UUID_NIL = "00000000-0000-0000-0000-000000000000"
 
 DEFAULT_FILE_NUMBER_LIMITS = 3

api/controllers/console/admin.py

@@ -56,7 +56,7 @@ class InsertExploreAppListApi(Resource):
         parser.add_argument("position", type=int, required=True, nullable=False, location="json")
         args = parser.parse_args()
 
-        app = db.session.execute(select(App).where(App.id == args["app_id"])).scalar_one_or_none()
+        app = db.session.execute(select(App).filter(App.id == args["app_id"])).scalar_one_or_none()
         if not app:
             raise NotFound(f"App '{args['app_id']}' is not found")
 
@@ -74,7 +74,7 @@ class InsertExploreAppListApi(Resource):
 
         with Session(db.engine) as session:
             recommended_app = session.execute(
-                select(RecommendedApp).where(RecommendedApp.app_id == args["app_id"])
+                select(RecommendedApp).filter(RecommendedApp.app_id == args["app_id"])
             ).scalar_one_or_none()
 
             if not recommended_app:
@@ -117,21 +117,21 @@ class InsertExploreAppApi(Resource):
     def delete(self, app_id):
         with Session(db.engine) as session:
             recommended_app = session.execute(
-                select(RecommendedApp).where(RecommendedApp.app_id == str(app_id))
+                select(RecommendedApp).filter(RecommendedApp.app_id == str(app_id))
             ).scalar_one_or_none()
 
         if not recommended_app:
             return {"result": "success"}, 204
 
         with Session(db.engine) as session:
-            app = session.execute(select(App).where(App.id == recommended_app.app_id)).scalar_one_or_none()
+            app = session.execute(select(App).filter(App.id == recommended_app.app_id)).scalar_one_or_none()
 
         if app:
             app.is_public = False
 
         with Session(db.engine) as session:
             installed_apps = session.execute(
-                select(InstalledApp).where(
+                select(InstalledApp).filter(
                     InstalledApp.app_id == recommended_app.app_id,
                     InstalledApp.tenant_id != InstalledApp.app_owner_tenant_id,
                 )

api/controllers/console/apikey.py

@@ -61,7 +61,7 @@ class BaseApiKeyListResource(Resource):
         _get_resource(resource_id, current_user.current_tenant_id, self.resource_model)
         keys = (
             db.session.query(ApiToken)
-            .where(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id)
+            .filter(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id)
             .all()
         )
         return {"items": keys}
@@ -76,7 +76,7 @@ class BaseApiKeyListResource(Resource):
 
         current_key_count = (
             db.session.query(ApiToken)
-            .where(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id)
+            .filter(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id)
             .count()
         )
 
@@ -117,7 +117,7 @@ class BaseApiKeyResource(Resource):
 
         key = (
             db.session.query(ApiToken)
-            .where(
+            .filter(
                 getattr(ApiToken, self.resource_id_field) == resource_id,
                 ApiToken.type == self.resource_type,
                 ApiToken.id == api_key_id,
@@ -128,7 +128,7 @@ class BaseApiKeyResource(Resource):
         if key is None:
             flask_restful.abort(404, message="API key not found")
 
-        db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
+        db.session.query(ApiToken).filter(ApiToken.id == api_key_id).delete()
         db.session.commit()
 
         return {"result": "success"}, 204

api/controllers/console/app/app.py

@@ -151,7 +151,6 @@ class AppApi(Resource):
         parser.add_argument("icon", type=str, location="json")
         parser.add_argument("icon_background", type=str, location="json")
         parser.add_argument("use_icon_as_answer_icon", type=bool, location="json")
-        parser.add_argument("max_active_requests", type=int, location="json")
         args = parser.parse_args()
 
         app_service = AppService()

api/controllers/console/app/conversation.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 
 import pytz  # pip install pytz
 from flask_login import current_user
@@ -19,7 +19,6 @@ from fields.conversation_fields import (
     conversation_pagination_fields,
     conversation_with_summary_pagination_fields,
 )
-from libs.datetime_utils import naive_utc_now
 from libs.helper import DatetimeString
 from libs.login import login_required
 from models import Conversation, EndUser, Message, MessageAnnotation
@@ -49,7 +48,7 @@ class CompletionConversationApi(Resource):
         query = db.select(Conversation).where(Conversation.app_id == app_model.id, Conversation.mode == "completion")
 
         if args["keyword"]:
-            query = query.join(Message, Message.conversation_id == Conversation.id).where(
+            query = query.join(Message, Message.conversation_id == Conversation.id).filter(
                 or_(
                     Message.query.ilike("%{}%".format(args["keyword"])),
                     Message.answer.ilike("%{}%".format(args["keyword"])),
@@ -121,7 +120,7 @@ class CompletionConversationDetailApi(Resource):
 
         conversation = (
             db.session.query(Conversation)
-            .where(Conversation.id == conversation_id, Conversation.app_id == app_model.id)
+            .filter(Conversation.id == conversation_id, Conversation.app_id == app_model.id)
             .first()
         )
 
@@ -181,7 +180,7 @@ class ChatConversationApi(Resource):
                     Message.conversation_id == Conversation.id,
                 )
                 .join(subquery, subquery.c.conversation_id == Conversation.id)
-                .where(
+                .filter(
                     or_(
                         Message.query.ilike(keyword_filter),
                         Message.answer.ilike(keyword_filter),
@@ -286,7 +285,7 @@ class ChatConversationDetailApi(Resource):
 
         conversation = (
             db.session.query(Conversation)
-            .where(Conversation.id == conversation_id, Conversation.app_id == app_model.id)
+            .filter(Conversation.id == conversation_id, Conversation.app_id == app_model.id)
             .first()
         )
 
@@ -308,7 +307,7 @@ api.add_resource(ChatConversationDetailApi, "/apps/<uuid:app_id>/chat-conversati
 def _get_conversation(app_model, conversation_id):
     conversation = (
         db.session.query(Conversation)
-        .where(Conversation.id == conversation_id, Conversation.app_id == app_model.id)
+        .filter(Conversation.id == conversation_id, Conversation.app_id == app_model.id)
         .first()
     )
 
@@ -316,7 +315,7 @@ def _get_conversation(app_model, conversation_id):
         raise NotFound("Conversation Not Exists.")
 
     if not conversation.read_at:
-        conversation.read_at = naive_utc_now()
+        conversation.read_at = datetime.now(UTC).replace(tzinfo=None)
         conversation.read_account_id = current_user.id
         db.session.commit()
 

api/controllers/console/app/mcp_server.py

@@ -26,7 +26,7 @@ class AppMCPServerController(Resource):
     @get_app_model
     @marshal_with(app_server_fields)
     def get(self, app_model):
-        server = db.session.query(AppMCPServer).where(AppMCPServer.app_id == app_model.id).first()
+        server = db.session.query(AppMCPServer).filter(AppMCPServer.app_id == app_model.id).first()
         return server
 
     @setup_required
@@ -35,20 +35,16 @@ class AppMCPServerController(Resource):
     @get_app_model
     @marshal_with(app_server_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be editor, admin, or owner
         if not current_user.is_editor:
             raise NotFound()
         parser = reqparse.RequestParser()
-        parser.add_argument("description", type=str, required=False, location="json")
+        parser.add_argument("description", type=str, required=True, location="json")
         parser.add_argument("parameters", type=dict, required=True, location="json")
         args = parser.parse_args()
-
-        description = args.get("description")
-        if not description:
-            description = app_model.description or ""
-
         server = AppMCPServer(
             name=app_model.name,
-            description=description,
+            description=args["description"],
             parameters=json.dumps(args["parameters"], ensure_ascii=False),
             status=AppMCPServerStatus.ACTIVE,
             app_id=app_model.id,
@@ -69,22 +65,14 @@ class AppMCPServerController(Resource):
             raise NotFound()
         parser = reqparse.RequestParser()
         parser.add_argument("id", type=str, required=True, location="json")
-        parser.add_argument("description", type=str, required=False, location="json")
+        parser.add_argument("description", type=str, required=True, location="json")
         parser.add_argument("parameters", type=dict, required=True, location="json")
         parser.add_argument("status", type=str, required=False, location="json")
         args = parser.parse_args()
-        server = db.session.query(AppMCPServer).where(AppMCPServer.id == args["id"]).first()
+        server = db.session.query(AppMCPServer).filter(AppMCPServer.id == args["id"]).first()
         if not server:
             raise NotFound()
-
+        server.description = args["description"]
-        description = args.get("description")
-        if description is None:
-            pass
-        elif not description:
-            server.description = app_model.description or ""
-        else:
-            server.description = description
-
         server.parameters = json.dumps(args["parameters"], ensure_ascii=False)
         if args["status"]:
             if args["status"] not in [status.value for status in AppMCPServerStatus]:
@@ -102,12 +90,7 @@ class AppMCPServerRefreshController(Resource):
     def get(self, server_id):
         if not current_user.is_editor:
             raise NotFound()
-        server = (
+        server = db.session.query(AppMCPServer).filter(AppMCPServer.id == server_id).first()
-            db.session.query(AppMCPServer)
-            .where(AppMCPServer.id == server_id)
-            .where(AppMCPServer.tenant_id == current_user.current_tenant_id)
-            .first()
-        )
         if not server:
             raise NotFound()
         server.server_code = AppMCPServer.generate_server_code(16)

api/controllers/console/app/message.py

@@ -5,7 +5,6 @@ from flask_restful import Resource, fields, marshal_with, reqparse
 from flask_restful.inputs import int_range
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
-import services
 from controllers.console import api
 from controllers.console.app.error import (
     CompletionRequestError,
@@ -28,7 +27,7 @@ from fields.conversation_fields import annotation_fields, message_detail_fields
 from libs.helper import uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from libs.login import login_required
-from models.model import AppMode, Conversation, Message, MessageAnnotation
+from models.model import AppMode, Conversation, Message, MessageAnnotation, MessageFeedback
 from services.annotation_service import AppAnnotationService
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
@@ -56,7 +55,7 @@ class ChatMessageListApi(Resource):
 
         conversation = (
             db.session.query(Conversation)
-            .where(Conversation.id == args["conversation_id"], Conversation.app_id == app_model.id)
+            .filter(Conversation.id == args["conversation_id"], Conversation.app_id == app_model.id)
             .first()
         )
 
@@ -66,7 +65,7 @@ class ChatMessageListApi(Resource):
         if args["first_id"]:
             first_message = (
                 db.session.query(Message)
-                .where(Message.conversation_id == conversation.id, Message.id == args["first_id"])
+                .filter(Message.conversation_id == conversation.id, Message.id == args["first_id"])
                 .first()
             )
 
@@ -75,7 +74,7 @@ class ChatMessageListApi(Resource):
 
             history_messages = (
                 db.session.query(Message)
-                .where(
+                .filter(
                     Message.conversation_id == conversation.id,
                     Message.created_at < first_message.created_at,
                     Message.id != first_message.id,
@@ -87,7 +86,7 @@ class ChatMessageListApi(Resource):
         else:
             history_messages = (
                 db.session.query(Message)
-                .where(Message.conversation_id == conversation.id)
+                .filter(Message.conversation_id == conversation.id)
                 .order_by(Message.created_at.desc())
                 .limit(args["limit"])
                 .all()
@@ -98,7 +97,7 @@ class ChatMessageListApi(Resource):
             current_page_first_message = history_messages[-1]
             rest_count = (
                 db.session.query(Message)
-                .where(
+                .filter(
                     Message.conversation_id == conversation.id,
                     Message.created_at < current_page_first_message.created_at,
                     Message.id != current_page_first_message.id,
@@ -125,17 +124,34 @@ class MessageFeedbackApi(Resource):
         parser.add_argument("rating", type=str, choices=["like", "dislike", None], location="json")
         args = parser.parse_args()
 
-        try:
+        message_id = str(args["message_id"])
-            MessageService.create_feedback(
+
-                app_model=app_model,
+        message = db.session.query(Message).filter(Message.id == message_id, Message.app_id == app_model.id).first()
-                message_id=str(args["message_id"]),
+
-                user=current_user,
+        if not message:
-                rating=args.get("rating"),
-                content=None,
-            )
-        except services.errors.message.MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
+        feedback = message.admin_feedback
+
+        if not args["rating"] and feedback:
+            db.session.delete(feedback)
+        elif args["rating"] and feedback:
+            feedback.rating = args["rating"]
+        elif not args["rating"] and not feedback:
+            raise ValueError("rating cannot be None when feedback not exists")
+        else:
+            feedback = MessageFeedback(
+                app_id=app_model.id,
+                conversation_id=message.conversation_id,
+                message_id=message.id,
+                rating=args["rating"],
+                from_source="admin",
+                from_account_id=current_user.id,
+            )
+            db.session.add(feedback)
+
+        db.session.commit()
+
         return {"result": "success"}
 
 
@@ -167,7 +183,7 @@ class MessageAnnotationCountApi(Resource):
     @account_initialization_required
     @get_app_model
     def get(self, app_model):
-        count = db.session.query(MessageAnnotation).where(MessageAnnotation.app_id == app_model.id).count()
+        count = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app_model.id).count()
 
         return {"count": count}
 
@@ -214,7 +230,7 @@ class MessageApi(Resource):
     def get(self, app_model, message_id):
         message_id = str(message_id)
 
-        message = db.session.query(Message).where(Message.id == message_id, Message.app_id == app_model.id).first()
+        message = db.session.query(Message).filter(Message.id == message_id, Message.app_id == app_model.id).first()
 
         if not message:
             raise NotFound("Message Not Exists.")

api/controllers/console/app/model_config.py

@@ -42,7 +42,7 @@ class ModelConfigResource(Resource):
         if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
             # get original app model config
             original_app_model_config = (
-                db.session.query(AppModelConfig).where(AppModelConfig.id == app_model.app_model_config_id).first()
+                db.session.query(AppModelConfig).filter(AppModelConfig.id == app_model.app_model_config_id).first()
             )
             if original_app_model_config is None:
                 raise ValueError("Original app model config not found")

api/controllers/console/app/site.py

@@ -1,3 +1,5 @@
+from datetime import UTC, datetime
+
 from flask_login import current_user
 from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
@@ -8,7 +10,6 @@ from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
-from libs.datetime_utils import naive_utc_now
 from libs.login import login_required
 from models import Site
 
@@ -49,7 +50,7 @@ class AppSite(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
-        site = db.session.query(Site).where(Site.app_id == app_model.id).first()
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
         if not site:
             raise NotFound
 
@@ -76,7 +77,7 @@ class AppSite(Resource):
                 setattr(site, attr_name, value)
 
         site.updated_by = current_user.id
-        site.updated_at = naive_utc_now()
+        site.updated_at = datetime.now(UTC).replace(tzinfo=None)
         db.session.commit()
 
         return site
@@ -93,14 +94,14 @@ class AppSiteAccessTokenReset(Resource):
         if not current_user.is_admin_or_owner:
             raise Forbidden()
 
-        site = db.session.query(Site).where(Site.app_id == app_model.id).first()
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
 
         if not site:
             raise NotFound
 
         site.code = Site.generate_code(16)
         site.updated_by = current_user.id
-        site.updated_at = naive_utc_now()
+        site.updated_at = datetime.now(UTC).replace(tzinfo=None)
         db.session.commit()
 
         return site

api/controllers/console/app/statistic.py

@@ -2,7 +2,6 @@ from datetime import datetime
 from decimal import Decimal
 
 import pytz
-import sqlalchemy as sa
 from flask import jsonify
 from flask_login import current_user
 from flask_restful import Resource, reqparse
@@ -10,11 +9,10 @@ from flask_restful import Resource, reqparse
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
-from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required
-from models import AppMode, Message
+from models.model import AppMode
 
 
 class DailyMessageStatistic(Resource):
@@ -87,41 +85,46 @@ class DailyConversationStatistic(Resource):
         parser.add_argument("end", type=DatetimeString("%Y-%m-%d %H:%M"), location="args")
         args = parser.parse_args()
 
+        sql_query = """SELECT
+    DATE(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date,
+    COUNT(DISTINCT messages.conversation_id) AS conversation_count
+FROM
+    messages
+WHERE
+    app_id = :app_id"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
 
-        stmt = (
-            sa.select(
-                sa.func.date(
-                    sa.func.date_trunc("day", sa.text("created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz"))
-                ).label("date"),
-                sa.func.count(sa.distinct(Message.conversation_id)).label("conversation_count"),
-            )
-            .select_from(Message)
-            .where(Message.app_id == app_model.id, Message.invoke_from != InvokeFrom.DEBUGGER.value)
-        )
-
         if args["start"]:
             start_datetime = datetime.strptime(args["start"], "%Y-%m-%d %H:%M")
             start_datetime = start_datetime.replace(second=0)
+
             start_datetime_timezone = timezone.localize(start_datetime)
             start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
-            stmt = stmt.where(Message.created_at >= start_datetime_utc)
+
+            sql_query += " AND created_at >= :start"
+            arg_dict["start"] = start_datetime_utc
 
         if args["end"]:
             end_datetime = datetime.strptime(args["end"], "%Y-%m-%d %H:%M")
             end_datetime = end_datetime.replace(second=0)
+
             end_datetime_timezone = timezone.localize(end_datetime)
             end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
-            stmt = stmt.where(Message.created_at < end_datetime_utc)
 
-        stmt = stmt.group_by("date").order_by("date")
+            sql_query += " AND created_at < :end"
+            arg_dict["end"] = end_datetime_utc
+
+        sql_query += " GROUP BY date ORDER BY date"
 
         response_data = []
+
         with db.engine.begin() as conn:
-            rs = conn.execute(stmt, {"tz": account.timezone})
+            rs = conn.execute(db.text(sql_query), arg_dict)
-            for row in rs:
+            for i in rs:
-                response_data.append({"date": str(row.date), "conversation_count": row.conversation_count})
+                response_data.append({"date": str(i.date), "conversation_count": i.conversation_count})
 
         return jsonify({"data": response_data})
 

api/controllers/console/app/workflow_draft_variable.py

@@ -68,18 +68,13 @@ def _create_pagination_parser():
     return parser
 
 
-def _serialize_variable_type(workflow_draft_var: WorkflowDraftVariable) -> str:
-    value_type = workflow_draft_var.value_type
-    return value_type.exposed_type().value
-
-
 _WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS = {
     "id": fields.String,
     "type": fields.String(attribute=lambda model: model.get_variable_type()),
     "name": fields.String,
     "description": fields.String,
     "selector": fields.List(fields.String, attribute=lambda model: model.get_selector()),
-    "value_type": fields.String(attribute=_serialize_variable_type),
+    "value_type": fields.String,
     "edited": fields.Boolean(attribute=lambda model: model.edited),
     "visible": fields.Boolean,
 }
@@ -95,7 +90,7 @@ _WORKFLOW_DRAFT_ENV_VARIABLE_FIELDS = {
     "name": fields.String,
     "description": fields.String,
     "selector": fields.List(fields.String, attribute=lambda model: model.get_selector()),
-    "value_type": fields.String(attribute=_serialize_variable_type),
+    "value_type": fields.String,
     "edited": fields.Boolean(attribute=lambda model: model.edited),
     "visible": fields.Boolean,
 }
@@ -401,7 +396,7 @@ class EnvironmentVariableCollectionApi(Resource):
                     "name": v.name,
                     "description": v.description,
                     "selector": v.selector,
-                    "value_type": v.value_type.exposed_type().value,
+                    "value_type": v.value_type.value,
                     "value": v.value,
                     # Do not track edited for env vars.
                     "edited": False,

api/controllers/console/app/wraps.py

@@ -11,7 +11,7 @@ from models import App, AppMode
 def _load_app_model(app_id: str) -> Optional[App]:
     app_model = (
         db.session.query(App)
-        .where(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
+        .filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
         .first()
     )
     return app_model
@@ -35,6 +35,8 @@ def get_app_model(view: Optional[Callable] = None, *, mode: Union[AppMode, list[
                 raise AppNotFoundError()
 
             app_mode = AppMode.value_of(app_model.mode)
+            if app_mode == AppMode.CHANNEL:
+                raise AppNotFoundError()
 
             if mode is not None:
                 if isinstance(mode, list):

api/controllers/console/auth/activate.py

@@ -1,3 +1,5 @@
+import datetime
+
 from flask import request
 from flask_restful import Resource, reqparse
 
@@ -5,7 +7,6 @@ from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from libs.datetime_utils import naive_utc_now
 from libs.helper import StrLen, email, extract_remote_ip, timezone
 from models.account import AccountStatus
 from services.account_service import AccountService, RegisterService
@@ -64,7 +65,7 @@ class ActivateApi(Resource):
         account.timezone = args["timezone"]
         account.interface_theme = "light"
         account.status = AccountStatus.ACTIVE.value
-        account.initialized_at = naive_utc_now()
+        account.initialized_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
         db.session.commit()
 
         token_pair = AccountService.login(account, ip_address=extract_remote_ip(request))

api/controllers/console/auth/error.py

@@ -27,19 +27,7 @@ class InvalidTokenError(BaseHTTPException):
 
 class PasswordResetRateLimitExceededError(BaseHTTPException):
     error_code = "password_reset_rate_limit_exceeded"
-    description = "Too many password reset emails have been sent. Please try again in 1 minute."
+    description = "Too many password reset emails have been sent. Please try again in 1 minutes."
-    code = 429
-
-
-class EmailChangeRateLimitExceededError(BaseHTTPException):
-    error_code = "email_change_rate_limit_exceeded"
-    description = "Too many email change emails have been sent. Please try again in 1 minute."
-    code = 429
-
-
-class OwnerTransferRateLimitExceededError(BaseHTTPException):
-    error_code = "owner_transfer_rate_limit_exceeded"
-    description = "Too many owner transfer emails have been sent. Please try again in 1 minute."
     code = 429
 
 
@@ -77,45 +65,3 @@ class EmailPasswordResetLimitError(BaseHTTPException):
     error_code = "email_password_reset_limit"
     description = "Too many failed password reset attempts. Please try again in 24 hours."
     code = 429
-
-
-class EmailChangeLimitError(BaseHTTPException):
-    error_code = "email_change_limit"
-    description = "Too many failed email change attempts. Please try again in 24 hours."
-    code = 429
-
-
-class EmailAlreadyInUseError(BaseHTTPException):
-    error_code = "email_already_in_use"
-    description = "A user with this email already exists."
-    code = 400
-
-
-class OwnerTransferLimitError(BaseHTTPException):
-    error_code = "owner_transfer_limit"
-    description = "Too many failed owner transfer attempts. Please try again in 24 hours."
-    code = 429
-
-
-class NotOwnerError(BaseHTTPException):
-    error_code = "not_owner"
-    description = "You are not the owner of the workspace."
-    code = 400
-
-
-class CannotTransferOwnerToSelfError(BaseHTTPException):
-    error_code = "cannot_transfer_owner_to_self"
-    description = "You cannot transfer ownership to yourself."
-    code = 400
-
-
-class MemberNotInTenantError(BaseHTTPException):
-    error_code = "member_not_in_tenant"
-    description = "The member is not in the workspace."
-    code = 400
-
-
-class AccountInFreezeError(BaseHTTPException):
-    error_code = "account_in_freeze"
-    description = "This email is temporarily unavailable."
-    code = 400

api/controllers/console/auth/oauth.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import UTC, datetime
 from typing import Optional
 
 import requests
@@ -12,7 +13,6 @@ from configs import dify_config
 from constants.languages import languages
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
-from libs.datetime_utils import naive_utc_now
 from libs.helper import extract_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models import Account
@@ -110,7 +110,7 @@ class OAuthCallback(Resource):
 
         if account.status == AccountStatus.PENDING.value:
             account.status = AccountStatus.ACTIVE.value
-            account.initialized_at = naive_utc_now()
+            account.initialized_at = datetime.now(UTC).replace(tzinfo=None)
             db.session.commit()
 
         try:

api/controllers/console/datasets/data_source.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 
 from flask import request
@@ -14,7 +15,6 @@ from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.notion_extractor import NotionExtractor
 from extensions.ext_database import db
 from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
-from libs.datetime_utils import naive_utc_now
 from libs.login import login_required
 from models import DataSourceOauthBinding, Document
 from services.dataset_service import DatasetService, DocumentService
@@ -30,7 +30,7 @@ class DataSourceApi(Resource):
         # get workspace data source integrates
         data_source_integrates = (
             db.session.query(DataSourceOauthBinding)
-            .where(
+            .filter(
                 DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
                 DataSourceOauthBinding.disabled == False,
             )
@@ -88,7 +88,7 @@ class DataSourceApi(Resource):
         if action == "enable":
             if data_source_binding.disabled:
                 data_source_binding.disabled = False
-                data_source_binding.updated_at = naive_utc_now()
+                data_source_binding.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
                 db.session.add(data_source_binding)
                 db.session.commit()
             else:
@@ -97,7 +97,7 @@ class DataSourceApi(Resource):
         if action == "disable":
             if not data_source_binding.disabled:
                 data_source_binding.disabled = True
-                data_source_binding.updated_at = naive_utc_now()
+                data_source_binding.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
                 db.session.add(data_source_binding)
                 db.session.commit()
             else:
@@ -171,7 +171,7 @@ class DataSourceNotionApi(Resource):
         page_id = str(page_id)
         with Session(db.engine) as session:
             data_source_binding = session.execute(
-                select(DataSourceOauthBinding).where(
+                select(DataSourceOauthBinding).filter(
                     db.and_(
                         DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
                         DataSourceOauthBinding.provider == "notion",

api/controllers/console/datasets/datasets.py

@@ -211,6 +211,10 @@ class DatasetApi(Resource):
         else:
             data["embedding_available"] = True
 
+        if data.get("permission") == "partial_members":
+            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+            data.update({"partial_member_list": part_users_list})
+
         return data, 200
 
     @setup_required
@@ -412,7 +416,7 @@ class DatasetIndexingEstimateApi(Resource):
             file_ids = args["info_list"]["file_info_list"]["file_ids"]
             file_details = (
                 db.session.query(UploadFile)
-                .where(UploadFile.tenant_id == current_user.current_tenant_id, UploadFile.id.in_(file_ids))
+                .filter(UploadFile.tenant_id == current_user.current_tenant_id, UploadFile.id.in_(file_ids))
                 .all()
             )
 
@@ -517,14 +521,14 @@ class DatasetIndexingStatusApi(Resource):
         dataset_id = str(dataset_id)
         documents = (
             db.session.query(Document)
-            .where(Document.dataset_id == dataset_id, Document.tenant_id == current_user.current_tenant_id)
+            .filter(Document.dataset_id == dataset_id, Document.tenant_id == current_user.current_tenant_id)
             .all()
         )
         documents_status = []
         for document in documents:
             completed_segments = (
                 db.session.query(DocumentSegment)
-                .where(
+                .filter(
                     DocumentSegment.completed_at.isnot(None),
                     DocumentSegment.document_id == str(document.id),
                     DocumentSegment.status != "re_segment",
@@ -533,7 +537,7 @@ class DatasetIndexingStatusApi(Resource):
             )
             total_segments = (
                 db.session.query(DocumentSegment)
-                .where(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
                 .count()
             )
             # Create a dictionary with document attributes and additional fields
@@ -568,7 +572,7 @@ class DatasetApiKeyApi(Resource):
     def get(self):
         keys = (
             db.session.query(ApiToken)
-            .where(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_user.current_tenant_id)
+            .filter(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_user.current_tenant_id)
             .all()
         )
         return {"items": keys}
@@ -584,7 +588,7 @@ class DatasetApiKeyApi(Resource):
 
         current_key_count = (
             db.session.query(ApiToken)
-            .where(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_user.current_tenant_id)
+            .filter(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_user.current_tenant_id)
             .count()
         )
 
@@ -620,7 +624,7 @@ class DatasetApiDeleteApi(Resource):
 
         key = (
             db.session.query(ApiToken)
-            .where(
+            .filter(
                 ApiToken.tenant_id == current_user.current_tenant_id,
                 ApiToken.type == self.resource_type,
                 ApiToken.id == api_key_id,
@@ -631,7 +635,7 @@ class DatasetApiDeleteApi(Resource):
         if key is None:
             flask_restful.abort(404, message="API key not found")
 
-        db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
+        db.session.query(ApiToken).filter(ApiToken.id == api_key_id).delete()
         db.session.commit()
 
         return {"result": "success"}, 204

api/controllers/console/datasets/datasets_document.py

@@ -1,5 +1,6 @@
 import logging
 from argparse import ArgumentTypeError
+from datetime import UTC, datetime
 from typing import cast
 
 from flask import request
@@ -48,7 +49,6 @@ from fields.document_fields import (
     document_status_fields,
     document_with_segments_fields,
 )
-from libs.datetime_utils import naive_utc_now
 from libs.login import login_required
 from models import Dataset, DatasetProcessRule, Document, DocumentSegment, UploadFile
 from services.dataset_service import DatasetService, DocumentService
@@ -124,7 +124,7 @@ class GetProcessRuleApi(Resource):
             # get the latest process rule
             dataset_process_rule = (
                 db.session.query(DatasetProcessRule)
-                .where(DatasetProcessRule.dataset_id == document.dataset_id)
+                .filter(DatasetProcessRule.dataset_id == document.dataset_id)
                 .order_by(DatasetProcessRule.created_at.desc())
                 .limit(1)
                 .one_or_none()
@@ -176,7 +176,7 @@ class DatasetDocumentListApi(Resource):
 
         if search:
             search = f"%{search}%"
-            query = query.where(Document.name.like(search))
+            query = query.filter(Document.name.like(search))
 
         if sort.startswith("-"):
             sort_logic = desc
@@ -212,7 +212,7 @@ class DatasetDocumentListApi(Resource):
             for document in documents:
                 completed_segments = (
                     db.session.query(DocumentSegment)
-                    .where(
+                    .filter(
                         DocumentSegment.completed_at.isnot(None),
                         DocumentSegment.document_id == str(document.id),
                         DocumentSegment.status != "re_segment",
@@ -221,7 +221,7 @@ class DatasetDocumentListApi(Resource):
                 )
                 total_segments = (
                     db.session.query(DocumentSegment)
-                    .where(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                    .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
                     .count()
                 )
                 document.completed_segments = completed_segments
@@ -417,7 +417,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
 
                 file = (
                     db.session.query(UploadFile)
-                    .where(UploadFile.tenant_id == document.tenant_id, UploadFile.id == file_id)
+                    .filter(UploadFile.tenant_id == document.tenant_id, UploadFile.id == file_id)
                     .first()
                 )
 
@@ -492,7 +492,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 file_id = data_source_info["upload_file_id"]
                 file_detail = (
                     db.session.query(UploadFile)
-                    .where(UploadFile.tenant_id == current_user.current_tenant_id, UploadFile.id == file_id)
+                    .filter(UploadFile.tenant_id == current_user.current_tenant_id, UploadFile.id == file_id)
                     .first()
                 )
 
@@ -568,7 +568,7 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
         for document in documents:
             completed_segments = (
                 db.session.query(DocumentSegment)
-                .where(
+                .filter(
                     DocumentSegment.completed_at.isnot(None),
                     DocumentSegment.document_id == str(document.id),
                     DocumentSegment.status != "re_segment",
@@ -577,7 +577,7 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
             )
             total_segments = (
                 db.session.query(DocumentSegment)
-                .where(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
                 .count()
             )
             # Create a dictionary with document attributes and additional fields
@@ -611,7 +611,7 @@ class DocumentIndexingStatusApi(DocumentResource):
 
         completed_segments = (
             db.session.query(DocumentSegment)
-            .where(
+            .filter(
                 DocumentSegment.completed_at.isnot(None),
                 DocumentSegment.document_id == str(document_id),
                 DocumentSegment.status != "re_segment",
@@ -620,7 +620,7 @@ class DocumentIndexingStatusApi(DocumentResource):
         )
         total_segments = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.document_id == str(document_id), DocumentSegment.status != "re_segment")
+            .filter(DocumentSegment.document_id == str(document_id), DocumentSegment.status != "re_segment")
             .count()
         )
 
@@ -750,7 +750,7 @@ class DocumentProcessingApi(DocumentResource):
                 raise InvalidActionError("Document not in indexing state.")
 
             document.paused_by = current_user.id
-            document.paused_at = naive_utc_now()
+            document.paused_at = datetime.now(UTC).replace(tzinfo=None)
             document.is_paused = True
             db.session.commit()
 
@@ -830,7 +830,7 @@ class DocumentMetadataApi(DocumentResource):
                     document.doc_metadata[key] = value
 
         document.doc_type = doc_type
-        document.updated_at = naive_utc_now()
+        document.updated_at = datetime.now(UTC).replace(tzinfo=None)
         db.session.commit()
 
         return {"result": "success", "message": "Document metadata updated."}, 200

api/controllers/console/datasets/datasets_segments.py

@@ -78,7 +78,7 @@ class DatasetDocumentSegmentListApi(Resource):
 
         query = (
             select(DocumentSegment)
-            .where(
+            .filter(
                 DocumentSegment.document_id == str(document_id),
                 DocumentSegment.tenant_id == current_user.current_tenant_id,
             )
@@ -86,19 +86,19 @@ class DatasetDocumentSegmentListApi(Resource):
         )
 
         if status_list:
-            query = query.where(DocumentSegment.status.in_(status_list))
+            query = query.filter(DocumentSegment.status.in_(status_list))
 
         if hit_count_gte is not None:
-            query = query.where(DocumentSegment.hit_count >= hit_count_gte)
+            query = query.filter(DocumentSegment.hit_count >= hit_count_gte)
 
         if keyword:
             query = query.where(DocumentSegment.content.ilike(f"%{keyword}%"))
 
         if args["enabled"].lower() != "all":
             if args["enabled"].lower() == "true":
-                query = query.where(DocumentSegment.enabled == True)
+                query = query.filter(DocumentSegment.enabled == True)
             elif args["enabled"].lower() == "false":
-                query = query.where(DocumentSegment.enabled == False)
+                query = query.filter(DocumentSegment.enabled == False)
 
         segments = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False)
 
@@ -285,7 +285,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -331,7 +331,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -436,7 +436,7 @@ class ChildChunkAddApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -493,7 +493,7 @@ class ChildChunkAddApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -540,7 +540,7 @@ class ChildChunkAddApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -586,7 +586,7 @@ class ChildChunkUpdateApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -595,7 +595,7 @@ class ChildChunkUpdateApi(Resource):
         child_chunk_id = str(child_chunk_id)
         child_chunk = (
             db.session.query(ChildChunk)
-            .where(ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id)
+            .filter(ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not child_chunk:
@@ -635,7 +635,7 @@ class ChildChunkUpdateApi(Resource):
         segment_id = str(segment_id)
         segment = (
             db.session.query(DocumentSegment)
-            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not segment:
@@ -644,7 +644,7 @@ class ChildChunkUpdateApi(Resource):
         child_chunk_id = str(child_chunk_id)
         child_chunk = (
             db.session.query(ChildChunk)
-            .where(ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id)
+            .filter(ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id)
             .first()
         )
         if not child_chunk:

api/controllers/console/datasets/error.py

@@ -25,6 +25,12 @@ class UnsupportedFileTypeError(BaseHTTPException):
     code = 415
 
 
+class HighQualityDatasetOnlyError(BaseHTTPException):
+    error_code = "high_quality_dataset_only"
+    description = "Current operation only supports 'high-quality' datasets."
+    code = 400
+
+
 class DatasetNotInitializedError(BaseHTTPException):
     error_code = "dataset_not_initialized"
     description = "The dataset is still being initialized or indexing. Please wait a moment."

api/controllers/console/datasets/website.py

@@ -4,7 +4,7 @@ from controllers.console import api
 from controllers.console.datasets.error import WebsiteCrawlError
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
-from services.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteService
+from services.website_service import WebsiteService
 
 
 class WebsiteCrawlApi(Resource):
@@ -24,16 +24,10 @@ class WebsiteCrawlApi(Resource):
         parser.add_argument("url", type=str, required=True, nullable=True, location="json")
         parser.add_argument("options", type=dict, required=True, nullable=True, location="json")
         args = parser.parse_args()
-
+        WebsiteService.document_create_args_validate(args)
-        # Create typed request and validate
+        # crawl url
-        try:
-            api_request = WebsiteCrawlApiRequest.from_args(args)
-        except ValueError as e:
-            raise WebsiteCrawlError(str(e))
-
-        # Crawl URL using typed request
         try:
-            result = WebsiteService.crawl_url(api_request)
+            result = WebsiteService.crawl_url(args)
         except Exception as e:
             raise WebsiteCrawlError(str(e))
         return result, 200
@@ -49,16 +43,9 @@ class WebsiteCrawlStatusApi(Resource):
             "provider", type=str, choices=["firecrawl", "watercrawl", "jinareader"], required=True, location="args"
         )
         args = parser.parse_args()
-
+        # get crawl status
-        # Create typed request and validate
-        try:
-            api_request = WebsiteCrawlStatusApiRequest.from_args(args, job_id)
-        except ValueError as e:
-            raise WebsiteCrawlError(str(e))
-
-        # Get crawl status using typed request
         try:
-            result = WebsiteService.get_crawl_status_typed(api_request)
+            result = WebsiteService.get_crawl_status(job_id, args["provider"])
         except Exception as e:
             raise WebsiteCrawlError(str(e))
         return result, 200

api/controllers/console/explore/completion.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import UTC, datetime
 
 from flask_login import current_user
 from flask_restful import reqparse
@@ -26,7 +27,6 @@ from core.errors.error import (
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
 from libs import helper
-from libs.datetime_utils import naive_utc_now
 from libs.helper import uuid_value
 from models.model import AppMode
 from services.app_generate_service import AppGenerateService
@@ -51,7 +51,7 @@ class CompletionApi(InstalledAppResource):
         streaming = args["response_mode"] == "streaming"
         args["auto_generate_name"] = False
 
-        installed_app.last_used_at = naive_utc_now()
+        installed_app.last_used_at = datetime.now(UTC).replace(tzinfo=None)
         db.session.commit()
 
         try:
@@ -111,7 +111,7 @@ class ChatApi(InstalledAppResource):
 
         args["auto_generate_name"] = False
 
-        installed_app.last_used_at = naive_utc_now()
+        installed_app.last_used_at = datetime.now(UTC).replace(tzinfo=None)
         db.session.commit()
 
         try:

api/controllers/console/explore/installed_app.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import UTC, datetime
 from typing import Any
 
 from flask import request
@@ -12,7 +13,6 @@ from controllers.console.explore.wraps import InstalledAppResource
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from fields.installed_app_fields import installed_app_list_fields
-from libs.datetime_utils import naive_utc_now
 from libs.login import login_required
 from models import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
@@ -34,11 +34,11 @@ class InstalledAppsListApi(Resource):
         if app_id:
             installed_apps = (
                 db.session.query(InstalledApp)
-                .where(and_(InstalledApp.tenant_id == current_tenant_id, InstalledApp.app_id == app_id))
+                .filter(and_(InstalledApp.tenant_id == current_tenant_id, InstalledApp.app_id == app_id))
                 .all()
             )
         else:
-            installed_apps = db.session.query(InstalledApp).where(InstalledApp.tenant_id == current_tenant_id).all()
+            installed_apps = db.session.query(InstalledApp).filter(InstalledApp.tenant_id == current_tenant_id).all()
 
         current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
         installed_app_list: list[dict[str, Any]] = [
@@ -94,12 +94,12 @@ class InstalledAppsListApi(Resource):
         parser.add_argument("app_id", type=str, required=True, help="Invalid app_id")
         args = parser.parse_args()
 
-        recommended_app = db.session.query(RecommendedApp).where(RecommendedApp.app_id == args["app_id"]).first()
+        recommended_app = db.session.query(RecommendedApp).filter(RecommendedApp.app_id == args["app_id"]).first()
         if recommended_app is None:
             raise NotFound("App not found")
 
         current_tenant_id = current_user.current_tenant_id
-        app = db.session.query(App).where(App.id == args["app_id"]).first()
+        app = db.session.query(App).filter(App.id == args["app_id"]).first()
 
         if app is None:
             raise NotFound("App not found")
@@ -109,7 +109,7 @@ class InstalledAppsListApi(Resource):
 
         installed_app = (
             db.session.query(InstalledApp)
-            .where(and_(InstalledApp.app_id == args["app_id"], InstalledApp.tenant_id == current_tenant_id))
+            .filter(and_(InstalledApp.app_id == args["app_id"], InstalledApp.tenant_id == current_tenant_id))
             .first()
         )
 
@@ -122,7 +122,7 @@ class InstalledAppsListApi(Resource):
                 tenant_id=current_tenant_id,
                 app_owner_tenant_id=app.tenant_id,
                 is_pinned=False,
-                last_used_at=naive_utc_now(),
+                last_used_at=datetime.now(UTC).replace(tzinfo=None),
             )
             db.session.add(new_installed_app)
             db.session.commit()

api/controllers/console/explore/wraps.py

@@ -28,7 +28,7 @@ def installed_app_required(view=None):
 
             installed_app = (
                 db.session.query(InstalledApp)
-                .where(
+                .filter(
                     InstalledApp.id == str(installed_app_id), InstalledApp.tenant_id == current_user.current_tenant_id
                 )
                 .first()

api/controllers/console/workspace/__init__.py

@@ -21,7 +21,7 @@ def plugin_permission_required(
             with Session(db.engine) as session:
                 permission = (
                     session.query(TenantPluginPermission)
-                    .where(
+                    .filter(
                         TenantPluginPermission.tenant_id == tenant_id,
                     )
                     .first()

api/controllers/console/workspace/account.py

@@ -1,22 +1,13 @@
+import datetime
+
 import pytz
 from flask import request
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with, reqparse
-from sqlalchemy import select
-from sqlalchemy.orm import Session
 
 from configs import dify_config
 from constants.languages import supported_language
 from controllers.console import api
-from controllers.console.auth.error import (
-    AccountInFreezeError,
-    EmailAlreadyInUseError,
-    EmailChangeLimitError,
-    EmailCodeError,
-    InvalidEmailError,
-    InvalidTokenError,
-)
-from controllers.console.error import AccountNotFound, EmailSendIpLimitError
 from controllers.console.workspace.error import (
     AccountAlreadyInitedError,
     CurrentPasswordIncorrectError,
@@ -27,18 +18,15 @@ from controllers.console.workspace.error import (
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_enabled,
-    enable_change_email,
     enterprise_license_required,
     only_edition_cloud,
     setup_required,
 )
 from extensions.ext_database import db
 from fields.member_fields import account_fields
-from libs.datetime_utils import naive_utc_now
+from libs.helper import TimestampField, timezone
-from libs.helper import TimestampField, email, extract_remote_ip, timezone
 from libs.login import login_required
 from models import AccountIntegrate, InvitationCode
-from models.account import Account
 from services.account_service import AccountService
 from services.billing_service import BillingService
 from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError
@@ -69,7 +57,7 @@ class AccountInitApi(Resource):
             # check invitation code
             invitation_code = (
                 db.session.query(InvitationCode)
-                .where(
+                .filter(
                     InvitationCode.code == args["invitation_code"],
                     InvitationCode.status == "unused",
                 )
@@ -80,7 +68,7 @@ class AccountInitApi(Resource):
                 raise InvalidInvitationCodeError()
 
             invitation_code.status = "used"
-            invitation_code.used_at = naive_utc_now()
+            invitation_code.used_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
             invitation_code.used_by_tenant_id = account.current_tenant_id
             invitation_code.used_by_account_id = account.id
 
@@ -88,7 +76,7 @@ class AccountInitApi(Resource):
         account.timezone = args["timezone"]
         account.interface_theme = "light"
         account.status = "active"
-        account.initialized_at = naive_utc_now()
+        account.initialized_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
         db.session.commit()
 
         return {"result": "success"}
@@ -229,7 +217,7 @@ class AccountIntegrateApi(Resource):
     def get(self):
         account = current_user
 
-        account_integrates = db.session.query(AccountIntegrate).where(AccountIntegrate.account_id == account.id).all()
+        account_integrates = db.session.query(AccountIntegrate).filter(AccountIntegrate.account_id == account.id).all()
 
         base_url = request.url_root.rstrip("/")
         oauth_base_path = "/console/api/oauth/login"
@@ -381,141 +369,6 @@ class EducationAutoCompleteApi(Resource):
         return BillingService.EducationIdentity.autocomplete(args["keywords"], args["page"], args["limit"])
 
 
-class ChangeEmailSendEmailApi(Resource):
-    @enable_change_email
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("language", type=str, required=False, location="json")
-        parser.add_argument("phase", type=str, required=False, location="json")
-        parser.add_argument("token", type=str, required=False, location="json")
-        args = parser.parse_args()
-
-        ip_address = extract_remote_ip(request)
-        if AccountService.is_email_send_ip_limit(ip_address):
-            raise EmailSendIpLimitError()
-
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
-        else:
-            language = "en-US"
-        account = None
-        user_email = args["email"]
-        if args["phase"] is not None and args["phase"] == "new_email":
-            if args["token"] is None:
-                raise InvalidTokenError()
-
-            reset_data = AccountService.get_change_email_data(args["token"])
-            if reset_data is None:
-                raise InvalidTokenError()
-            user_email = reset_data.get("email", "")
-
-            if user_email != current_user.email:
-                raise InvalidEmailError()
-        else:
-            with Session(db.engine) as session:
-                account = session.execute(select(Account).filter_by(email=args["email"])).scalar_one_or_none()
-            if account is None:
-                raise AccountNotFound()
-
-        token = AccountService.send_change_email_email(
-            account=account, email=args["email"], old_email=user_email, language=language, phase=args["phase"]
-        )
-        return {"result": "success", "data": token}
-
-
-class ChangeEmailCheckApi(Resource):
-    @enable_change_email
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("code", type=str, required=True, location="json")
-        parser.add_argument("token", type=str, required=True, nullable=False, location="json")
-        args = parser.parse_args()
-
-        user_email = args["email"]
-
-        is_change_email_error_rate_limit = AccountService.is_change_email_error_rate_limit(args["email"])
-        if is_change_email_error_rate_limit:
-            raise EmailChangeLimitError()
-
-        token_data = AccountService.get_change_email_data(args["token"])
-        if token_data is None:
-            raise InvalidTokenError()
-
-        if user_email != token_data.get("email"):
-            raise InvalidEmailError()
-
-        if args["code"] != token_data.get("code"):
-            AccountService.add_change_email_error_rate_limit(args["email"])
-            raise EmailCodeError()
-
-        # Verified, revoke the first token
-        AccountService.revoke_change_email_token(args["token"])
-
-        # Refresh token data by generating a new token
-        _, new_token = AccountService.generate_change_email_token(
-            user_email, code=args["code"], old_email=token_data.get("old_email"), additional_data={}
-        )
-
-        AccountService.reset_change_email_error_rate_limit(args["email"])
-        return {"is_valid": True, "email": token_data.get("email"), "token": new_token}
-
-
-class ChangeEmailResetApi(Resource):
-    @enable_change_email
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @marshal_with(account_fields)
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("new_email", type=email, required=True, location="json")
-        parser.add_argument("token", type=str, required=True, nullable=False, location="json")
-        args = parser.parse_args()
-
-        if AccountService.is_account_in_freeze(args["new_email"]):
-            raise AccountInFreezeError()
-
-        if not AccountService.check_email_unique(args["new_email"]):
-            raise EmailAlreadyInUseError()
-
-        reset_data = AccountService.get_change_email_data(args["token"])
-        if not reset_data:
-            raise InvalidTokenError()
-
-        AccountService.revoke_change_email_token(args["token"])
-
-        old_email = reset_data.get("old_email", "")
-        if current_user.email != old_email:
-            raise AccountNotFound()
-
-        updated_account = AccountService.update_account(current_user, email=args["new_email"])
-
-        AccountService.send_change_email_completed_notify_email(
-            email=args["new_email"],
-        )
-
-        return updated_account
-
-
-class CheckEmailUnique(Resource):
-    @setup_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("email", type=email, required=True, location="json")
-        args = parser.parse_args()
-        if not AccountService.check_email_unique(args["email"]):
-            raise EmailAlreadyInUseError()
-        return {"result": "success"}
-
-
 # Register API resources
 api.add_resource(AccountInitApi, "/account/init")
 api.add_resource(AccountProfileApi, "/account/profile")
@@ -532,10 +385,5 @@ api.add_resource(AccountDeleteUpdateFeedbackApi, "/account/delete/feedback")
 api.add_resource(EducationVerifyApi, "/account/education/verify")
 api.add_resource(EducationApi, "/account/education")
 api.add_resource(EducationAutoCompleteApi, "/account/education/autocomplete")
-# Change email
-api.add_resource(ChangeEmailSendEmailApi, "/account/change-email")
-api.add_resource(ChangeEmailCheckApi, "/account/change-email/validity")
-api.add_resource(ChangeEmailResetApi, "/account/change-email/reset")
-api.add_resource(CheckEmailUnique, "/account/change-email/check-email-unique")
 # api.add_resource(AccountEmailApi, '/account/email')
 # api.add_resource(AccountEmailVerifyApi, '/account/email-verify')

api/controllers/console/workspace/error.py

@@ -13,6 +13,12 @@ class CurrentPasswordIncorrectError(BaseHTTPException):
     code = 400
 
 
+class ProviderRequestFailedError(BaseHTTPException):
+    error_code = "provider_request_failed"
+    description = None
+    code = 400
+
+
 class InvalidInvitationCodeError(BaseHTTPException):
     error_code = "invalid_invitation_code"
     description = "Invalid invitation code."

api/controllers/console/workspace/members.py

@@ -1,34 +1,22 @@
 from urllib import parse
 
-from flask import request
 from flask_login import current_user
 from flask_restful import Resource, abort, marshal_with, reqparse
 
 import services
 from configs import dify_config
 from controllers.console import api
-from controllers.console.auth.error import (
+from controllers.console.error import WorkspaceMembersLimitExceeded
-    CannotTransferOwnerToSelfError,
-    EmailCodeError,
-    InvalidEmailError,
-    InvalidTokenError,
-    MemberNotInTenantError,
-    NotOwnerError,
-    OwnerTransferLimitError,
-)
-from controllers.console.error import EmailSendIpLimitError, WorkspaceMembersLimitExceeded
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_resource_check,
-    is_allow_transfer_owner,
     setup_required,
 )
 from extensions.ext_database import db
 from fields.member_fields import account_with_role_list_fields
-from libs.helper import extract_remote_ip
 from libs.login import login_required
 from models.account import Account, TenantAccountRole
-from services.account_service import AccountService, RegisterService, TenantService
+from services.account_service import RegisterService, TenantService
 from services.errors.account import AccountAlreadyInTenantError
 from services.feature_service import FeatureService
 
@@ -108,7 +96,7 @@ class MemberCancelInviteApi(Resource):
     @login_required
     @account_initialization_required
     def delete(self, member_id):
-        member = db.session.query(Account).where(Account.id == str(member_id)).first()
+        member = db.session.query(Account).filter(Account.id == str(member_id)).first()
         if member is None:
             abort(404)
         else:
@@ -168,146 +156,8 @@ class DatasetOperatorMemberListApi(Resource):
         return {"result": "success", "accounts": members}, 200
 
 
-class SendOwnerTransferEmailApi(Resource):
-    """Send owner transfer email."""
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @is_allow_transfer_owner
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("language", type=str, required=False, location="json")
-        args = parser.parse_args()
-        ip_address = extract_remote_ip(request)
-        if AccountService.is_email_send_ip_limit(ip_address):
-            raise EmailSendIpLimitError()
-
-        # check if the current user is the owner of the workspace
-        if not TenantService.is_owner(current_user, current_user.current_tenant):
-            raise NotOwnerError()
-
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
-        else:
-            language = "en-US"
-
-        email = current_user.email
-
-        token = AccountService.send_owner_transfer_email(
-            account=current_user,
-            email=email,
-            language=language,
-            workspace_name=current_user.current_tenant.name,
-        )
-
-        return {"result": "success", "data": token}
-
-
-class OwnerTransferCheckApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @is_allow_transfer_owner
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("code", type=str, required=True, location="json")
-        parser.add_argument("token", type=str, required=True, nullable=False, location="json")
-        args = parser.parse_args()
-        # check if the current user is the owner of the workspace
-        if not TenantService.is_owner(current_user, current_user.current_tenant):
-            raise NotOwnerError()
-
-        user_email = current_user.email
-
-        is_owner_transfer_error_rate_limit = AccountService.is_owner_transfer_error_rate_limit(user_email)
-        if is_owner_transfer_error_rate_limit:
-            raise OwnerTransferLimitError()
-
-        token_data = AccountService.get_owner_transfer_data(args["token"])
-        if token_data is None:
-            raise InvalidTokenError()
-
-        if user_email != token_data.get("email"):
-            raise InvalidEmailError()
-
-        if args["code"] != token_data.get("code"):
-            AccountService.add_owner_transfer_error_rate_limit(user_email)
-            raise EmailCodeError()
-
-        # Verified, revoke the first token
-        AccountService.revoke_owner_transfer_token(args["token"])
-
-        # Refresh token data by generating a new token
-        _, new_token = AccountService.generate_owner_transfer_token(user_email, code=args["code"], additional_data={})
-
-        AccountService.reset_owner_transfer_error_rate_limit(user_email)
-        return {"is_valid": True, "email": token_data.get("email"), "token": new_token}
-
-
-class OwnerTransfer(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @is_allow_transfer_owner
-    def post(self, member_id):
-        parser = reqparse.RequestParser()
-        parser.add_argument("token", type=str, required=True, nullable=False, location="json")
-        args = parser.parse_args()
-
-        # check if the current user is the owner of the workspace
-        if not TenantService.is_owner(current_user, current_user.current_tenant):
-            raise NotOwnerError()
-
-        if current_user.id == str(member_id):
-            raise CannotTransferOwnerToSelfError()
-
-        transfer_token_data = AccountService.get_owner_transfer_data(args["token"])
-        if not transfer_token_data:
-            raise InvalidTokenError()
-
-        if transfer_token_data.get("email") != current_user.email:
-            raise InvalidEmailError()
-
-        AccountService.revoke_owner_transfer_token(args["token"])
-
-        member = db.session.get(Account, str(member_id))
-        if not member:
-            abort(404)
-        else:
-            member_account = member
-        if not TenantService.is_member(member_account, current_user.current_tenant):
-            raise MemberNotInTenantError()
-
-        try:
-            assert member is not None, "Member not found"
-            TenantService.update_member_role(current_user.current_tenant, member, "owner", current_user)
-
-            AccountService.send_new_owner_transfer_notify_email(
-                account=member,
-                email=member.email,
-                workspace_name=current_user.current_tenant.name,
-            )
-
-            AccountService.send_old_owner_transfer_notify_email(
-                account=current_user,
-                email=current_user.email,
-                workspace_name=current_user.current_tenant.name,
-                new_owner_email=member.email,
-            )
-
-        except Exception as e:
-            raise ValueError(str(e))
-
-        return {"result": "success"}
-
-
 api.add_resource(MemberListApi, "/workspaces/current/members")
 api.add_resource(MemberInviteEmailApi, "/workspaces/current/members/invite-email")
 api.add_resource(MemberCancelInviteApi, "/workspaces/current/members/<uuid:member_id>")
 api.add_resource(MemberUpdateRoleApi, "/workspaces/current/members/<uuid:member_id>/update-role")
 api.add_resource(DatasetOperatorMemberListApi, "/workspaces/current/dataset-operators")
-# owner transfer
-api.add_resource(SendOwnerTransferEmailApi, "/workspaces/current/members/send-owner-transfer-confirm-email")
-api.add_resource(OwnerTransferCheckApi, "/workspaces/current/members/owner-transfer-check")
-api.add_resource(OwnerTransfer, "/workspaces/current/members/<uuid:member_id>/owner-transfer")

api/controllers/console/workspace/tool_providers.py

@@ -1,35 +1,26 @@
 import io
 from urllib.parse import urlparse
 
-from flask import make_response, redirect, request, send_file
+from flask import redirect, send_file
 from flask_login import current_user
-from flask_restful import (
+from flask_restful import Resource, reqparse
-    Resource,
+from sqlalchemy.orm import Session
-    reqparse,
-)
 from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
 from controllers.console import api
-from controllers.console.wraps import (
+from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
-    account_initialization_required,
-    enterprise_license_required,
-    setup_required,
-)
 from core.mcp.auth.auth_flow import auth, handle_callback
 from core.mcp.auth.auth_provider import OAuthClientProvider
 from core.mcp.error import MCPAuthError, MCPError
 from core.mcp.mcp_client import MCPClient
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.plugin.entities.plugin import ToolProviderID
+from extensions.ext_database import db
-from core.plugin.impl.oauth import OAuthHandler
+from libs.helper import alphanumeric, uuid_value
-from core.tools.entities.tool_entities import CredentialType
-from libs.helper import StrLen, alphanumeric, uuid_value
 from libs.login import login_required
-from services.plugin.oauth_service import OAuthProxyService
 from services.tools.api_tools_manage_service import ApiToolManageService
 from services.tools.builtin_tools_manage_service import BuiltinToolManageService
-from services.tools.mcp_tools_manage_service import MCPToolManageService
+from services.tools.mcp_tools_mange_service import MCPToolManageService
 from services.tools.tool_labels_service import ToolLabelsService
 from services.tools.tools_manage_service import ToolCommonService
 from services.tools.tools_transform_service import ToolTransformService
@@ -98,7 +89,7 @@ class ToolBuiltinProviderInfoApi(Resource):
         user_id = user.id
         tenant_id = user.current_tenant_id
 
-        return jsonable_encoder(BuiltinToolManageService.get_builtin_tool_provider_info(tenant_id, provider))
+        return jsonable_encoder(BuiltinToolManageService.get_builtin_tool_provider_info(user_id, tenant_id, provider))
 
 
 class ToolBuiltinProviderDeleteApi(Resource):
@@ -107,47 +98,17 @@ class ToolBuiltinProviderDeleteApi(Resource):
     @account_initialization_required
     def post(self, provider):
         user = current_user
+
         if not user.is_admin_or_owner:
             raise Forbidden()
 
+        user_id = user.id
         tenant_id = user.current_tenant_id
-        req = reqparse.RequestParser()
-        req.add_argument("credential_id", type=str, required=True, nullable=False, location="json")
-        args = req.parse_args()
 
         return BuiltinToolManageService.delete_builtin_tool_provider(
+            user_id,
             tenant_id,
             provider,
-            args["credential_id"],
-        )
-
-
-class ToolBuiltinProviderAddApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, provider):
-        user = current_user
-
-        user_id = user.id
-        tenant_id = user.current_tenant_id
-
-        parser = reqparse.RequestParser()
-        parser.add_argument("credentials", type=dict, required=True, nullable=False, location="json")
-        parser.add_argument("name", type=StrLen(30), required=False, nullable=False, location="json")
-        parser.add_argument("type", type=str, required=True, nullable=False, location="json")
-        args = parser.parse_args()
-
-        if args["type"] not in CredentialType.values():
-            raise ValueError(f"Invalid credential type: {args['type']}")
-
-        return BuiltinToolManageService.add_builtin_tool_provider(
-            user_id=user_id,
-            tenant_id=tenant_id,
-            provider=provider,
-            credentials=args["credentials"],
-            name=args["name"],
-            api_type=CredentialType.of(args["type"]),
         )
 
 
@@ -165,20 +126,19 @@ class ToolBuiltinProviderUpdateApi(Resource):
         tenant_id = user.current_tenant_id
 
         parser = reqparse.RequestParser()
-        parser.add_argument("credential_id", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("credentials", type=dict, required=True, nullable=False, location="json")
-        parser.add_argument("credentials", type=dict, required=False, nullable=True, location="json")
-        parser.add_argument("name", type=StrLen(30), required=False, nullable=True, location="json")
 
         args = parser.parse_args()
 
-        result = BuiltinToolManageService.update_builtin_tool_provider(
+        with Session(db.engine) as session:
-            user_id=user_id,
+            result = BuiltinToolManageService.update_builtin_tool_provider(
-            tenant_id=tenant_id,
+                session=session,
-            provider=provider,
+                user_id=user_id,
-            credential_id=args["credential_id"],
+                tenant_id=tenant_id,
-            credentials=args.get("credentials", None),
+                provider_name=provider,
-            name=args.get("name", ""),
+                credentials=args["credentials"],
-        )
+            )
+            session.commit()
         return result
 
 
@@ -189,11 +149,9 @@ class ToolBuiltinProviderGetCredentialsApi(Resource):
     def get(self, provider):
         tenant_id = current_user.current_tenant_id
 
-        return jsonable_encoder(
+        return BuiltinToolManageService.get_builtin_tool_provider_credentials(
-            BuiltinToolManageService.get_builtin_tool_provider_credentials(
+            tenant_id=tenant_id,
-                tenant_id=tenant_id,
+            provider_name=provider,
-                provider_name=provider,
-            )
         )
 
 
@@ -386,15 +344,12 @@ class ToolBuiltinProviderCredentialsSchemaApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, provider, credential_type):
+    def get(self, provider):
         user = current_user
+
         tenant_id = user.current_tenant_id
 
-        return jsonable_encoder(
+        return BuiltinToolManageService.list_builtin_provider_credentials_schema(provider, tenant_id)
-            BuiltinToolManageService.list_builtin_provider_credentials_schema(
-                provider, CredentialType.of(credential_type), tenant_id
-            )
-        )
 
 
 class ToolApiProviderSchemaApi(Resource):
@@ -631,12 +586,15 @@ class ToolApiListApi(Resource):
     @account_initialization_required
     def get(self):
         user = current_user
+
+        user_id = user.id
         tenant_id = user.current_tenant_id
 
         return jsonable_encoder(
             [
                 provider.to_dict()
                 for provider in ApiToolManageService.list_api_tools(
+                    user_id,
                     tenant_id,
                 )
             ]
@@ -673,183 +631,6 @@ class ToolLabelsApi(Resource):
         return jsonable_encoder(ToolLabelsService.list_tool_labels())
 
 
-class ToolPluginOAuthApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, provider):
-        tool_provider = ToolProviderID(provider)
-        plugin_id = tool_provider.plugin_id
-        provider_name = tool_provider.provider_name
-
-        # todo check permission
-        user = current_user
-
-        if not user.is_admin_or_owner:
-            raise Forbidden()
-
-        tenant_id = user.current_tenant_id
-        oauth_client_params = BuiltinToolManageService.get_oauth_client(tenant_id=tenant_id, provider=provider)
-        if oauth_client_params is None:
-            raise Forbidden("no oauth available client config found for this tool provider")
-
-        oauth_handler = OAuthHandler()
-        context_id = OAuthProxyService.create_proxy_context(
-            user_id=current_user.id, tenant_id=tenant_id, plugin_id=plugin_id, provider=provider_name
-        )
-        redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider}/tool/callback"
-        authorization_url_response = oauth_handler.get_authorization_url(
-            tenant_id=tenant_id,
-            user_id=user.id,
-            plugin_id=plugin_id,
-            provider=provider_name,
-            redirect_uri=redirect_uri,
-            system_credentials=oauth_client_params,
-        )
-        response = make_response(jsonable_encoder(authorization_url_response))
-        response.set_cookie(
-            "context_id",
-            context_id,
-            httponly=True,
-            samesite="Lax",
-            max_age=OAuthProxyService.__MAX_AGE__,
-        )
-        return response
-
-
-class ToolOAuthCallback(Resource):
-    @setup_required
-    def get(self, provider):
-        context_id = request.cookies.get("context_id")
-        if not context_id:
-            raise Forbidden("context_id not found")
-
-        context = OAuthProxyService.use_proxy_context(context_id)
-        if context is None:
-            raise Forbidden("Invalid context_id")
-
-        tool_provider = ToolProviderID(provider)
-        plugin_id = tool_provider.plugin_id
-        provider_name = tool_provider.provider_name
-        user_id, tenant_id = context.get("user_id"), context.get("tenant_id")
-
-        oauth_handler = OAuthHandler()
-        oauth_client_params = BuiltinToolManageService.get_oauth_client(tenant_id, provider)
-        if oauth_client_params is None:
-            raise Forbidden("no oauth available client config found for this tool provider")
-
-        redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider}/tool/callback"
-        credentials_response = oauth_handler.get_credentials(
-            tenant_id=tenant_id,
-            user_id=user_id,
-            plugin_id=plugin_id,
-            provider=provider_name,
-            redirect_uri=redirect_uri,
-            system_credentials=oauth_client_params,
-            request=request,
-        )
-
-        credentials = credentials_response.credentials
-        expires_at = credentials_response.expires_at
-
-        if not credentials:
-            raise Exception("the plugin credentials failed")
-
-        # add credentials to database
-        BuiltinToolManageService.add_builtin_tool_provider(
-            user_id=user_id,
-            tenant_id=tenant_id,
-            provider=provider,
-            credentials=dict(credentials),
-            expires_at=expires_at,
-            api_type=CredentialType.OAUTH2,
-        )
-        return redirect(f"{dify_config.CONSOLE_WEB_URL}/oauth-callback")
-
-
-class ToolBuiltinProviderSetDefaultApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, provider):
-        parser = reqparse.RequestParser()
-        parser.add_argument("id", type=str, required=True, nullable=False, location="json")
-        args = parser.parse_args()
-        return BuiltinToolManageService.set_default_provider(
-            tenant_id=current_user.current_tenant_id, user_id=current_user.id, provider=provider, id=args["id"]
-        )
-
-
-class ToolOAuthCustomClient(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, provider):
-        parser = reqparse.RequestParser()
-        parser.add_argument("client_params", type=dict, required=False, nullable=True, location="json")
-        parser.add_argument("enable_oauth_custom_client", type=bool, required=False, nullable=True, location="json")
-        args = parser.parse_args()
-
-        user = current_user
-
-        if not user.is_admin_or_owner:
-            raise Forbidden()
-
-        return BuiltinToolManageService.save_custom_oauth_client_params(
-            tenant_id=user.current_tenant_id,
-            provider=provider,
-            client_params=args.get("client_params", {}),
-            enable_oauth_custom_client=args.get("enable_oauth_custom_client", True),
-        )
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, provider):
-        return jsonable_encoder(
-            BuiltinToolManageService.get_custom_oauth_client_params(
-                tenant_id=current_user.current_tenant_id, provider=provider
-            )
-        )
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def delete(self, provider):
-        return jsonable_encoder(
-            BuiltinToolManageService.delete_custom_oauth_client_params(
-                tenant_id=current_user.current_tenant_id, provider=provider
-            )
-        )
-
-
-class ToolBuiltinProviderGetOauthClientSchemaApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, provider):
-        return jsonable_encoder(
-            BuiltinToolManageService.get_builtin_tool_provider_oauth_client_schema(
-                tenant_id=current_user.current_tenant_id, provider_name=provider
-            )
-        )
-
-
-class ToolBuiltinProviderGetCredentialInfoApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, provider):
-        tenant_id = current_user.current_tenant_id
-
-        return jsonable_encoder(
-            BuiltinToolManageService.get_builtin_tool_provider_credential_info(
-                tenant_id=tenant_id,
-                provider=provider,
-            )
-        )
-
-
 class ToolProviderMCPApi(Resource):
     @setup_required
     @login_required
@@ -1013,33 +794,17 @@ class ToolMCPCallbackApi(Resource):
 # tool provider
 api.add_resource(ToolProviderListApi, "/workspaces/current/tool-providers")
 
-# tool oauth
-api.add_resource(ToolPluginOAuthApi, "/oauth/plugin/<path:provider>/tool/authorization-url")
-api.add_resource(ToolOAuthCallback, "/oauth/plugin/<path:provider>/tool/callback")
-api.add_resource(ToolOAuthCustomClient, "/workspaces/current/tool-provider/builtin/<path:provider>/oauth/custom-client")
-
 # builtin tool provider
 api.add_resource(ToolBuiltinProviderListToolsApi, "/workspaces/current/tool-provider/builtin/<path:provider>/tools")
 api.add_resource(ToolBuiltinProviderInfoApi, "/workspaces/current/tool-provider/builtin/<path:provider>/info")
-api.add_resource(ToolBuiltinProviderAddApi, "/workspaces/current/tool-provider/builtin/<path:provider>/add")
 api.add_resource(ToolBuiltinProviderDeleteApi, "/workspaces/current/tool-provider/builtin/<path:provider>/delete")
 api.add_resource(ToolBuiltinProviderUpdateApi, "/workspaces/current/tool-provider/builtin/<path:provider>/update")
-api.add_resource(
-    ToolBuiltinProviderSetDefaultApi, "/workspaces/current/tool-provider/builtin/<path:provider>/default-credential"
-)
-api.add_resource(
-    ToolBuiltinProviderGetCredentialInfoApi, "/workspaces/current/tool-provider/builtin/<path:provider>/credential/info"
-)
 api.add_resource(
     ToolBuiltinProviderGetCredentialsApi, "/workspaces/current/tool-provider/builtin/<path:provider>/credentials"
 )
 api.add_resource(
     ToolBuiltinProviderCredentialsSchemaApi,
-    "/workspaces/current/tool-provider/builtin/<path:provider>/credential/schema/<path:credential_type>",
+    "/workspaces/current/tool-provider/builtin/<path:provider>/credentials_schema",
-)
-api.add_resource(
-    ToolBuiltinProviderGetOauthClientSchemaApi,
-    "/workspaces/current/tool-provider/builtin/<path:provider>/oauth/client-schema",
 )
 api.add_resource(ToolBuiltinProviderIconApi, "/workspaces/current/tool-provider/builtin/<path:provider>/icon")
 

api/controllers/console/wraps.py

@@ -235,29 +235,3 @@ def email_password_login_enabled(view):
         abort(403)
 
     return decorated
-
-
-def enable_change_email(view):
-    @wraps(view)
-    def decorated(*args, **kwargs):
-        features = FeatureService.get_system_features()
-        if features.enable_change_email:
-            return view(*args, **kwargs)
-
-        # otherwise, return 403
-        abort(403)
-
-    return decorated
-
-
-def is_allow_transfer_owner(view):
-    @wraps(view)
-    def decorated(*args, **kwargs):
-        features = FeatureService.get_features(current_user.current_tenant_id)
-        if features.is_allow_transfer_workspace:
-            return view(*args, **kwargs)
-
-        # otherwise, return 403
-        abort(403)
-
-    return decorated

api/controllers/inner_api/plugin/plugin.py

@@ -175,7 +175,6 @@ class PluginInvokeToolApi(Resource):
                     provider=payload.provider,
                     tool_name=payload.tool,
                     tool_parameters=payload.tool_parameters,
-                    credential_id=payload.credential_id,
                 ),
             )
 

api/controllers/inner_api/plugin/wraps.py

@@ -22,7 +22,7 @@ def get_user(tenant_id: str, user_id: str | None) -> Account | EndUser:
                 user_id = "DEFAULT-USER"
 
             if user_id == "DEFAULT-USER":
-                user_model = session.query(EndUser).where(EndUser.session_id == "DEFAULT-USER").first()
+                user_model = session.query(EndUser).filter(EndUser.session_id == "DEFAULT-USER").first()
                 if not user_model:
                     user_model = EndUser(
                         tenant_id=tenant_id,
@@ -36,7 +36,7 @@ def get_user(tenant_id: str, user_id: str | None) -> Account | EndUser:
             else:
                 user_model = AccountService.load_user(user_id)
                 if not user_model:
-                    user_model = session.query(EndUser).where(EndUser.id == user_id).first()
+                    user_model = session.query(EndUser).filter(EndUser.id == user_id).first()
                 if not user_model:
                     raise ValueError("user not found")
     except Exception:
@@ -71,7 +71,7 @@ def get_user_tenant(view: Optional[Callable] = None):
             try:
                 tenant_model = (
                     db.session.query(Tenant)
-                    .where(
+                    .filter(
                         Tenant.id == tenant_id,
                     )
                     .first()

api/controllers/inner_api/wraps.py

@@ -55,7 +55,7 @@ def enterprise_inner_api_user_auth(view):
         if signature_base64 != token:
             return view(*args, **kwargs)
 
-        kwargs["user"] = db.session.query(EndUser).where(EndUser.id == user_id).first()
+        kwargs["user"] = db.session.query(EndUser).filter(EndUser.id == user_id).first()
 
         return view(*args, **kwargs)
 

api/controllers/mcp/mcp.py

@@ -30,7 +30,7 @@ class MCPAppApi(Resource):
 
         request_id = args.get("id")
 
-        server = db.session.query(AppMCPServer).where(AppMCPServer.server_code == server_code).first()
+        server = db.session.query(AppMCPServer).filter(AppMCPServer.server_code == server_code).first()
         if not server:
             return helper.compact_generate_response(
                 create_mcp_error_response(request_id, types.INVALID_REQUEST, "Server Not Found")
@@ -41,7 +41,7 @@ class MCPAppApi(Resource):
                 create_mcp_error_response(request_id, types.INVALID_REQUEST, "Server is not active")
             )
 
-        app = db.session.query(App).where(App.id == server.app_id).first()
+        app = db.session.query(App).filter(App.id == server.app_id).first()
         if not app:
             return helper.compact_generate_response(
                 create_mcp_error_response(request_id, types.INVALID_REQUEST, "App Not Found")

api/controllers/service_api/app/completion.py

@@ -1,6 +1,5 @@
 import logging
 
-from flask import request
 from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -24,7 +23,6 @@ from core.errors.error import (
     ProviderTokenNotInitError,
     QuotaExceededError,
 )
-from core.helper.trace_id_helper import get_external_trace_id
 from core.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
@@ -113,10 +111,6 @@ class ChatApi(Resource):
 
         args = parser.parse_args()
 
-        external_trace_id = get_external_trace_id(request)
-        if external_trace_id:
-            args["external_trace_id"] = external_trace_id
-
         streaming = args["response_mode"] == "streaming"
 
         try:

api/controllers/service_api/app/site.py

@@ -16,7 +16,7 @@ class AppSiteApi(Resource):
     @marshal_with(fields.site_fields)
     def get(self, app_model: App):
         """Retrieve app site info."""
-        site = db.session.query(Site).where(Site.app_id == app_model.id).first()
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
 
         if not site:
             raise Forbidden()

api/controllers/service_api/app/workflow.py

@@ -1,10 +1,9 @@
 import logging
 
 from dateutil.parser import isoparse
-from flask import request
 from flask_restful import Resource, fields, marshal_with, reqparse
 from flask_restful.inputs import int_range
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import InternalServerError
 
 from controllers.service_api import api
@@ -24,7 +23,6 @@ from core.errors.error import (
     ProviderTokenNotInitError,
     QuotaExceededError,
 )
-from core.helper.trace_id_helper import get_external_trace_id
 from core.model_runtime.errors.invoke import InvokeError
 from core.workflow.entities.workflow_execution import WorkflowExecutionStatus
 from extensions.ext_database import db
@@ -32,7 +30,7 @@ from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs import helper
 from libs.helper import TimestampField
 from models.model import App, AppMode, EndUser
-from repositories.factory import DifyAPIRepositoryFactory
+from models.workflow import WorkflowRun
 from services.app_generate_service import AppGenerateService
 from services.errors.llm import InvokeRateLimitError
 from services.workflow_app_service import WorkflowAppService
@@ -65,15 +63,7 @@ class WorkflowRunDetailApi(Resource):
         if app_mode not in [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]:
             raise NotWorkflowAppError()
 
-        # Use repository to get workflow run
+        workflow_run = db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_run_id).first()
-        session_maker = sessionmaker(bind=db.engine, expire_on_commit=False)
-        workflow_run_repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker)
-
-        workflow_run = workflow_run_repo.get_workflow_run_by_id(
-            tenant_id=app_model.tenant_id,
-            app_id=app_model.id,
-            run_id=workflow_run_id,
-        )
         return workflow_run
 
 
@@ -92,9 +82,7 @@ class WorkflowRunApi(Resource):
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
         args = parser.parse_args()
-        external_trace_id = get_external_trace_id(request)
+
-        if external_trace_id:
-            args["external_trace_id"] = external_trace_id
         streaming = args.get("response_mode") == "streaming"
 
         try:

api/controllers/service_api/dataset/document.py

@@ -63,7 +63,7 @@ class DocumentAddByTextApi(DatasetApiResource):
 
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -136,7 +136,7 @@ class DocumentUpdateByTextApi(DatasetApiResource):
         args = parser.parse_args()
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -206,7 +206,7 @@ class DocumentAddByFileApi(DatasetApiResource):
         # get dataset info
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -299,7 +299,7 @@ class DocumentUpdateByFileApi(DatasetApiResource):
         # get dataset info
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -367,7 +367,7 @@ class DocumentDeleteApi(DatasetApiResource):
         tenant_id = str(tenant_id)
 
         # get dataset info
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
 
         if not dataset:
             raise ValueError("Dataset does not exist.")
@@ -398,7 +398,7 @@ class DocumentListApi(DatasetApiResource):
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
         search = request.args.get("keyword", default=None, type=str)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -406,7 +406,7 @@ class DocumentListApi(DatasetApiResource):
 
         if search:
             search = f"%{search}%"
-            query = query.where(Document.name.like(search))
+            query = query.filter(Document.name.like(search))
 
         query = query.order_by(desc(Document.created_at), desc(Document.position))
 
@@ -430,7 +430,7 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         batch = str(batch)
         tenant_id = str(tenant_id)
         # get dataset
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # get documents
@@ -441,7 +441,7 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         for document in documents:
             completed_segments = (
                 db.session.query(DocumentSegment)
-                .where(
+                .filter(
                     DocumentSegment.completed_at.isnot(None),
                     DocumentSegment.document_id == str(document.id),
                     DocumentSegment.status != "re_segment",
@@ -450,7 +450,7 @@ class DocumentIndexingStatusApi(DatasetApiResource):
             )
             total_segments = (
                 db.session.query(DocumentSegment)
-                .where(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
                 .count()
             )
             # Create a dictionary with document attributes and additional fields

api/controllers/service_api/dataset/error.py

@@ -25,6 +25,12 @@ class UnsupportedFileTypeError(BaseHTTPException):
     code = 415
 
 
+class HighQualityDatasetOnlyError(BaseHTTPException):
+    error_code = "high_quality_dataset_only"
+    description = "Current operation only supports 'high-quality' datasets."
+    code = 400
+
+
 class DatasetNotInitializedError(BaseHTTPException):
     error_code = "dataset_not_initialized"
     description = "The dataset is still being initialized or indexing. Please wait a moment."

api/controllers/service_api/dataset/segment.py

@@ -42,7 +42,7 @@ class SegmentApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
@@ -89,7 +89,7 @@ class SegmentApi(DatasetApiResource):
         tenant_id = str(tenant_id)
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
@@ -146,7 +146,7 @@ class DatasetSegmentApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -170,7 +170,7 @@ class DatasetSegmentApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -216,7 +216,7 @@ class DatasetSegmentApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -246,7 +246,7 @@ class ChildChunkApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -296,7 +296,7 @@ class ChildChunkApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -343,7 +343,7 @@ class DatasetChildChunkApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -382,7 +382,7 @@ class DatasetChildChunkApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
 

api/controllers/service_api/dataset/upload_file.py

@@ -17,7 +17,7 @@ class UploadFileApi(DatasetApiResource):
         # check dataset
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
@@ -31,7 +31,7 @@ class UploadFileApi(DatasetApiResource):
         data_source_info = document.data_source_info_dict
         if data_source_info and "upload_file_id" in data_source_info:
             file_id = data_source_info["upload_file_id"]
-            upload_file = db.session.query(UploadFile).where(UploadFile.id == file_id).first()
+            upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()
             if not upload_file:
                 raise NotFound("UploadFile not found.")
         else:

api/controllers/service_api/wraps.py

@@ -1,6 +1,6 @@
 import time
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from enum import Enum
 from functools import wraps
 from typing import Optional
@@ -15,7 +15,6 @@ from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
 
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from libs.datetime_utils import naive_utc_now
 from libs.login import _get_user
 from models.account import Account, Tenant, TenantAccountJoin, TenantStatus
 from models.dataset import Dataset, RateLimitLog
@@ -44,7 +43,7 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio
         def decorated_view(*args, **kwargs):
             api_token = validate_and_get_api_token("app")
 
-            app_model = db.session.query(App).where(App.id == api_token.app_id).first()
+            app_model = db.session.query(App).filter(App.id == api_token.app_id).first()
             if not app_model:
                 raise Forbidden("The app no longer exists.")
 
@@ -54,7 +53,7 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio
             if not app_model.enable_api:
                 raise Forbidden("The app's API service has been disabled.")
 
-            tenant = db.session.query(Tenant).where(Tenant.id == app_model.tenant_id).first()
+            tenant = db.session.query(Tenant).filter(Tenant.id == app_model.tenant_id).first()
             if tenant is None:
                 raise ValueError("Tenant does not exist.")
             if tenant.status == TenantStatus.ARCHIVE:
@@ -62,15 +61,15 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio
 
             tenant_account_join = (
                 db.session.query(Tenant, TenantAccountJoin)
-                .where(Tenant.id == api_token.tenant_id)
+                .filter(Tenant.id == api_token.tenant_id)
-                .where(TenantAccountJoin.tenant_id == Tenant.id)
+                .filter(TenantAccountJoin.tenant_id == Tenant.id)
-                .where(TenantAccountJoin.role.in_(["owner"]))
+                .filter(TenantAccountJoin.role.in_(["owner"]))
-                .where(Tenant.status == TenantStatus.NORMAL)
+                .filter(Tenant.status == TenantStatus.NORMAL)
                 .one_or_none()
             )  # TODO: only owner information is required, so only one is returned.
             if tenant_account_join:
                 tenant, ta = tenant_account_join
-                account = db.session.query(Account).where(Account.id == ta.account_id).first()
+                account = db.session.query(Account).filter(Account.id == ta.account_id).first()
                 # Login admin
                 if account:
                     account.current_tenant = tenant
@@ -213,15 +212,15 @@ def validate_dataset_token(view=None):
             api_token = validate_and_get_api_token("dataset")
             tenant_account_join = (
                 db.session.query(Tenant, TenantAccountJoin)
-                .where(Tenant.id == api_token.tenant_id)
+                .filter(Tenant.id == api_token.tenant_id)
-                .where(TenantAccountJoin.tenant_id == Tenant.id)
+                .filter(TenantAccountJoin.tenant_id == Tenant.id)
-                .where(TenantAccountJoin.role.in_(["owner"]))
+                .filter(TenantAccountJoin.role.in_(["owner"]))
-                .where(Tenant.status == TenantStatus.NORMAL)
+                .filter(Tenant.status == TenantStatus.NORMAL)
                 .one_or_none()
             )  # TODO: only owner information is required, so only one is returned.
             if tenant_account_join:
                 tenant, ta = tenant_account_join
-                account = db.session.query(Account).where(Account.id == ta.account_id).first()
+                account = db.session.query(Account).filter(Account.id == ta.account_id).first()
                 # Login admin
                 if account:
                     account.current_tenant = tenant
@@ -257,7 +256,7 @@ def validate_and_get_api_token(scope: str | None = None):
     if auth_scheme != "bearer":
         raise Unauthorized("Authorization scheme must be 'Bearer'")
 
-    current_time = naive_utc_now()
+    current_time = datetime.now(UTC).replace(tzinfo=None)
     cutoff_time = current_time - timedelta(minutes=1)
     with Session(db.engine, expire_on_commit=False) as session:
         update_stmt = (
@@ -293,7 +292,7 @@ def create_or_update_end_user_for_user_id(app_model: App, user_id: Optional[str]
 
     end_user = (
         db.session.query(EndUser)
-        .where(
+        .filter(
             EndUser.tenant_id == app_model.tenant_id,
             EndUser.app_id == app_model.id,
             EndUser.session_id == user_id,
@@ -320,7 +319,7 @@ class DatasetApiResource(Resource):
     method_decorators = [validate_dataset_token]
 
     def get_dataset(self, dataset_id: str, tenant_id: str) -> Dataset:
-        dataset = db.session.query(Dataset).where(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id).first()
+        dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id, Dataset.tenant_id == tenant_id).first()
 
         if not dataset:
             raise NotFound("Dataset not found.")

api/controllers/web/passport.py

@@ -3,7 +3,6 @@ from datetime import UTC, datetime, timedelta
 
 from flask import request
 from flask_restful import Resource
-from sqlalchemy import func, select
 from werkzeug.exceptions import NotFound, Unauthorized
 
 from configs import dify_config
@@ -43,17 +42,17 @@ class PassportResource(Resource):
                 raise WebAppAuthRequiredError()
 
         # get site from db and check if it is normal
-        site = db.session.scalar(select(Site).where(Site.code == app_code, Site.status == "normal"))
+        site = db.session.query(Site).filter(Site.code == app_code, Site.status == "normal").first()
         if not site:
             raise NotFound()
         # get app from db and check if it is normal and enable_site
-        app_model = db.session.scalar(select(App).where(App.id == site.app_id))
+        app_model = db.session.query(App).filter(App.id == site.app_id).first()
         if not app_model or app_model.status != "normal" or not app_model.enable_site:
             raise NotFound()
 
         if user_id:
-            end_user = db.session.scalar(
+            end_user = (
-                select(EndUser).where(EndUser.app_id == app_model.id, EndUser.session_id == user_id)
+                db.session.query(EndUser).filter(EndUser.app_id == app_model.id, EndUser.session_id == user_id).first()
             )
 
             if end_user:
@@ -122,11 +121,11 @@ def exchange_token_for_existing_web_user(app_code: str, enterprise_user_decoded:
     if not user_auth_type:
         raise Unauthorized("Missing auth_type in the token.")
 
-    site = db.session.scalar(select(Site).where(Site.code == app_code, Site.status == "normal"))
+    site = db.session.query(Site).filter(Site.code == app_code, Site.status == "normal").first()
     if not site:
         raise NotFound()
 
-    app_model = db.session.scalar(select(App).where(App.id == site.app_id))
+    app_model = db.session.query(App).filter(App.id == site.app_id).first()
     if not app_model or app_model.status != "normal" or not app_model.enable_site:
         raise NotFound()
 
@@ -141,14 +140,16 @@ def exchange_token_for_existing_web_user(app_code: str, enterprise_user_decoded:
 
     end_user = None
     if end_user_id:
-        end_user = db.session.scalar(select(EndUser).where(EndUser.id == end_user_id))
+        end_user = db.session.query(EndUser).filter(EndUser.id == end_user_id).first()
     if session_id:
-        end_user = db.session.scalar(
+        end_user = (
-            select(EndUser).where(
+            db.session.query(EndUser)
+            .filter(
                 EndUser.session_id == session_id,
                 EndUser.tenant_id == app_model.tenant_id,
                 EndUser.app_id == app_model.id,
             )
+            .first()
         )
     if not end_user:
         if not session_id:
@@ -186,8 +187,8 @@ def _exchange_for_public_app_token(app_model, site, token_decoded):
     user_id = token_decoded.get("user_id")
     end_user = None
     if user_id:
-        end_user = db.session.scalar(
+        end_user = (
-            select(EndUser).where(EndUser.app_id == app_model.id, EndUser.session_id == user_id)
+            db.session.query(EndUser).filter(EndUser.app_id == app_model.id, EndUser.session_id == user_id).first()
         )
 
     if not end_user:
@@ -223,8 +224,6 @@ def generate_session_id():
     """
     while True:
         session_id = str(uuid.uuid4())
-        existing_count = db.session.scalar(
+        existing_count = db.session.query(EndUser).filter(EndUser.session_id == session_id).count()
-            select(func.count()).select_from(EndUser).where(EndUser.session_id == session_id)
-        )
         if existing_count == 0:
             return session_id

api/controllers/web/site.py

@@ -57,7 +57,7 @@ class AppSiteApi(WebApiResource):
     def get(self, app_model, end_user):
         """Retrieve app site info."""
         # get site
-        site = db.session.query(Site).where(Site.app_id == app_model.id).first()
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
 
         if not site:
             raise Forbidden()

api/controllers/web/wraps.py

@@ -3,7 +3,6 @@ from functools import wraps
 
 from flask import request
 from flask_restful import Resource
-from sqlalchemy import select
 from werkzeug.exceptions import BadRequest, NotFound, Unauthorized
 
 from controllers.web.error import WebAppAuthAccessDeniedError, WebAppAuthRequiredError
@@ -49,8 +48,8 @@ def decode_jwt_token():
         decoded = PassportService().verify(tk)
         app_code = decoded.get("app_code")
         app_id = decoded.get("app_id")
-        app_model = db.session.scalar(select(App).where(App.id == app_id))
+        app_model = db.session.query(App).filter(App.id == app_id).first()
-        site = db.session.scalar(select(Site).where(Site.code == app_code))
+        site = db.session.query(Site).filter(Site.code == app_code).first()
         if not app_model:
             raise NotFound()
         if not app_code or not site:
@@ -58,7 +57,7 @@ def decode_jwt_token():
         if app_model.enable_site is False:
             raise BadRequest("Site is disabled.")
         end_user_id = decoded.get("end_user_id")
-        end_user = db.session.scalar(select(EndUser).where(EndUser.id == end_user_id))
+        end_user = db.session.query(EndUser).filter(EndUser.id == end_user_id).first()
         if not end_user:
             raise NotFound()
 

api/core/agent/base_agent_runner.py

@@ -3,8 +3,6 @@ import logging
 import uuid
 from typing import Optional, Union, cast
 
-from sqlalchemy import select
-
 from core.agent.entities import AgentEntity, AgentToolEntity
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfig
@@ -99,7 +97,7 @@ class BaseAgentRunner(AppRunner):
         # get how many agent thoughts have been created
         self.agent_thought_count = (
             db.session.query(MessageAgentThought)
-            .where(
+            .filter(
                 MessageAgentThought.message_id == self.message.id,
             )
             .count()
@@ -336,7 +334,7 @@ class BaseAgentRunner(AppRunner):
         Save agent thought
         """
         updated_agent_thought = (
-            db.session.query(MessageAgentThought).where(MessageAgentThought.id == agent_thought.id).first()
+            db.session.query(MessageAgentThought).filter(MessageAgentThought.id == agent_thought.id).first()
         )
         if not updated_agent_thought:
             raise ValueError("agent thought not found")
@@ -419,15 +417,12 @@ class BaseAgentRunner(AppRunner):
             if isinstance(prompt_message, SystemPromptMessage):
                 result.append(prompt_message)
 
-        messages = (
+        messages: list[Message] = (
-            (
+            db.session.query(Message)
-                db.session.execute(
+            .filter(
-                    select(Message)
+                Message.conversation_id == self.message.conversation_id,
-                    .where(Message.conversation_id == self.message.conversation_id)
-                    .order_by(Message.created_at.desc())
-                )
             )
-            .scalars()
+            .order_by(Message.created_at.desc())
             .all()
         )
 
@@ -496,7 +491,7 @@ class BaseAgentRunner(AppRunner):
         return result
 
     def organize_agent_user_prompt(self, message: Message) -> UserPromptMessage:
-        files = db.session.query(MessageFile).where(MessageFile.message_id == message.id).all()
+        files = db.session.query(MessageFile).filter(MessageFile.message_id == message.id).all()
         if not files:
             return UserPromptMessage(content=message.query)
         if message.app_model_config:

api/core/agent/entities.py

@@ -16,7 +16,6 @@ class AgentToolEntity(BaseModel):
     tool_name: str
     tool_parameters: dict[str, Any] = Field(default_factory=dict)
     plugin_unique_identifier: str | None = None
-    credential_id: str | None = None
 
 
 class AgentPromptEntity(BaseModel):

api/core/agent/plugin_entities.py

@@ -41,7 +41,6 @@ class AgentStrategyParameter(PluginParameter):
         APP_SELECTOR = CommonParameterType.APP_SELECTOR.value
         MODEL_SELECTOR = CommonParameterType.MODEL_SELECTOR.value
         TOOLS_SELECTOR = CommonParameterType.TOOLS_SELECTOR.value
-        ANY = CommonParameterType.ANY.value
 
         # deprecated, should not use.
         SYSTEM_FILES = CommonParameterType.SYSTEM_FILES.value

api/core/agent/strategy/base.py

@@ -4,7 +4,6 @@ from typing import Any, Optional
 
 from core.agent.entities import AgentInvokeMessage
 from core.agent.plugin_entities import AgentStrategyParameter
-from core.plugin.entities.request import InvokeCredentials
 
 
 class BaseAgentStrategy(ABC):
@@ -19,12 +18,11 @@ class BaseAgentStrategy(ABC):
         conversation_id: Optional[str] = None,
         app_id: Optional[str] = None,
         message_id: Optional[str] = None,
-        credentials: Optional[InvokeCredentials] = None,
     ) -> Generator[AgentInvokeMessage, None, None]:
         """
         Invoke the agent strategy.
         """
-        yield from self._invoke(params, user_id, conversation_id, app_id, message_id, credentials)
+        yield from self._invoke(params, user_id, conversation_id, app_id, message_id)
 
     def get_parameters(self) -> Sequence[AgentStrategyParameter]:
         """
@@ -40,6 +38,5 @@ class BaseAgentStrategy(ABC):
         conversation_id: Optional[str] = None,
         app_id: Optional[str] = None,
         message_id: Optional[str] = None,
-        credentials: Optional[InvokeCredentials] = None,
     ) -> Generator[AgentInvokeMessage, None, None]:
         pass

api/core/agent/strategy/plugin.py

@@ -4,7 +4,6 @@ from typing import Any, Optional
 from core.agent.entities import AgentInvokeMessage
 from core.agent.plugin_entities import AgentStrategyEntity, AgentStrategyParameter
 from core.agent.strategy.base import BaseAgentStrategy
-from core.plugin.entities.request import InvokeCredentials, PluginInvokeContext
 from core.plugin.impl.agent import PluginAgentClient
 from core.plugin.utils.converter import convert_parameters_to_plugin_format
 
@@ -41,7 +40,6 @@ class PluginAgentStrategy(BaseAgentStrategy):
         conversation_id: Optional[str] = None,
         app_id: Optional[str] = None,
         message_id: Optional[str] = None,
-        credentials: Optional[InvokeCredentials] = None,
     ) -> Generator[AgentInvokeMessage, None, None]:
         """
         Invoke the agent strategy.
@@ -60,5 +58,4 @@ class PluginAgentStrategy(BaseAgentStrategy):
             conversation_id=conversation_id,
             app_id=app_id,
             message_id=message_id,
-            context=PluginInvokeContext(credentials=credentials or InvokeCredentials()),
         )

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -39,7 +39,6 @@ class AgentConfigManager:
                         "provider_id": tool["provider_id"],
                         "tool_name": tool["tool_name"],
                         "tool_parameters": tool.get("tool_parameters", {}),
-                        "credential_id": tool.get("credential_id", None),
                     }
 
                     agent_tools.append(AgentToolEntity(**agent_tool_properties))

api/core/app/apps/README.md

@@ -0,0 +1,48 @@
+## Guidelines for Database Connection Management in App Runner and Task Pipeline
+
+Due to the presence of tasks in App Runner that require long execution times, such as LLM generation and external requests, Flask-Sqlalchemy's strategy for database connection pooling is to allocate one connection (transaction) per request. This approach keeps a connection occupied even during non-DB tasks, leading to the inability to acquire new connections during high concurrency requests due to multiple long-running tasks.
+
+Therefore, the database operations in App Runner and Task Pipeline must ensure connections are closed immediately after use, and it's better to pass IDs rather than Model objects to avoid detach errors.
+
+Examples:
+
+1. Creating a new record:
+
+   ```python
+   app = App(id=1)
+   db.session.add(app)
+   db.session.commit()
+   db.session.refresh(app)  # Retrieve table default values, like created_at, cached in the app object, won't affect after close
+   
+   # Handle non-long-running tasks or store the content of the App instance in memory (via variable assignment).
+   
+   db.session.close()
+   
+   return app.id
+   ```
+
+2. Fetching a record from the table:
+
+   ```python
+   app = db.session.query(App).filter(App.id == app_id).first()
+    
+   created_at = app.created_at
+    
+   db.session.close()
+   
+   # Handle tasks (include long-running).
+   
+   ```
+
+3. Updating a table field:
+
+   ```python
+   app = db.session.query(App).filter(App.id == app_id).first()
+
+   app.updated_at = time.utcnow()
+   db.session.commit()
+   db.session.close()
+
+   return app_id
+   ```
+   

api/core/app/apps/advanced_chat/app_generator.py

@@ -7,8 +7,7 @@ from typing import Any, Literal, Optional, Union, overload
 
 from flask import Flask, current_app
 from pydantic import ValidationError
-from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
-from sqlalchemy.orm import Session, sessionmaker
 
 import contexts
 from configs import dify_config
@@ -18,17 +17,16 @@ from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig
 from core.app.apps.advanced_chat.app_runner import AdvancedChatAppRunner
 from core.app.apps.advanced_chat.generate_response_converter import AdvancedChatAppGenerateResponseConverter
 from core.app.apps.advanced_chat.generate_task_pipeline import AdvancedChatAppGenerateTaskPipeline
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.apps.message_based_app_generator import MessageBasedAppGenerator
 from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager
 from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom
 from core.app.entities.task_entities import ChatbotAppBlockingResponse, ChatbotAppStreamResponse
-from core.helper.trace_id_helper import extract_external_trace_id_from_args
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
 from core.ops.ops_trace_manager import TraceQueueManager
 from core.prompt.utils.get_thread_messages_length import get_thread_messages_length
-from core.repositories import DifyCoreRepositoryFactory
+from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository
+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository
 from core.workflow.repositories.draft_variable_repository import (
     DraftVariableSaverFactory,
 )
@@ -114,10 +112,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         query = query.replace("\x00", "")
         inputs = args["inputs"]
 
-        extras = {
+        extras = {"auto_generate_conversation_name": args.get("auto_generate_name", False)}
-            "auto_generate_conversation_name": args.get("auto_generate_name", False),
-            **extract_external_trace_id_from_args(args),
-        }
 
         # get conversation
         conversation = None
@@ -188,14 +183,14 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             workflow_triggered_from = WorkflowRunTriggeredFrom.DEBUGGING
         else:
             workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN
-        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
+        workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
             triggered_from=workflow_triggered_from,
         )
         # Create workflow node execution repository
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
+        workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
@@ -265,14 +260,14 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         # Create session factory
         session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
         # Create workflow execution(aka workflow run) repository
-        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
+        workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
             triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
         )
         # Create workflow node execution repository
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
+        workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
@@ -348,14 +343,14 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         # Create session factory
         session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
         # Create workflow execution(aka workflow run) repository
-        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
+        workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
             triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
         )
         # Create workflow node execution repository
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
+        workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
@@ -487,52 +482,21 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         """
 
         with preserve_flask_contexts(flask_app, context_vars=context):
-            # get conversation and message
+            try:
-            conversation = self._get_conversation(conversation_id)
+                # get conversation and message
-            message = self._get_message(message_id)
+                conversation = self._get_conversation(conversation_id)
-
+                message = self._get_message(message_id)
-            with Session(db.engine, expire_on_commit=False) as session:
+
-                workflow = session.scalar(
+                # chatbot app
-                    select(Workflow).where(
+                runner = AdvancedChatAppRunner(
-                        Workflow.tenant_id == application_generate_entity.app_config.tenant_id,
+                    application_generate_entity=application_generate_entity,
-                        Workflow.app_id == application_generate_entity.app_config.app_id,
+                    queue_manager=queue_manager,
-                        Workflow.id == application_generate_entity.app_config.workflow_id,
+                    conversation=conversation,
-                    )
+                    message=message,
+                    dialogue_count=self._dialogue_count,
+                    variable_loader=variable_loader,
                 )
-                if workflow is None:
-                    raise ValueError("Workflow not found")
-
-                # Determine system_user_id based on invocation source
-                is_external_api_call = application_generate_entity.invoke_from in {
-                    InvokeFrom.WEB_APP,
-                    InvokeFrom.SERVICE_API,
-                }
-
-                if is_external_api_call:
-                    # For external API calls, use end user's session ID
-                    end_user = session.scalar(select(EndUser).where(EndUser.id == application_generate_entity.user_id))
-                    system_user_id = end_user.session_id if end_user else ""
-                else:
-                    # For internal calls, use the original user ID
-                    system_user_id = application_generate_entity.user_id
-
-                app = session.scalar(select(App).where(App.id == application_generate_entity.app_config.app_id))
-                if app is None:
-                    raise ValueError("App not found")
-
-            runner = AdvancedChatAppRunner(
-                application_generate_entity=application_generate_entity,
-                queue_manager=queue_manager,
-                conversation=conversation,
-                message=message,
-                dialogue_count=self._dialogue_count,
-                variable_loader=variable_loader,
-                workflow=workflow,
-                system_user_id=system_user_id,
-                app=app,
-            )
 
-            try:
                 runner.run()
             except GenerateTaskStoppedError:
                 pass

api/core/app/apps/advanced_chat/app_runner.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Mapping
-from typing import Any, Optional, cast
+from typing import Any, cast
 
 from sqlalchemy import select
 from sqlalchemy.orm import Session
@@ -9,29 +9,21 @@ from configs import dify_config
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner
-from core.app.entities.app_invoke_entities import (
+from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom
-    AdvancedChatAppGenerateEntity,
-    AppGenerateEntity,
-    InvokeFrom,
-)
 from core.app.entities.queue_entities import (
     QueueAnnotationReplyEvent,
     QueueStopEvent,
     QueueTextChunkEvent,
 )
-from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature
 from core.moderation.base import ModerationError
-from core.moderation.input_moderation import InputModeration
-from core.variables.variables import VariableUnion
 from core.workflow.callbacks import WorkflowCallback, WorkflowLoggingCallback
 from core.workflow.entities.variable_pool import VariablePool
-from core.workflow.system_variable import SystemVariable
+from core.workflow.enums import SystemVariableKey
 from core.workflow.variable_loader import VariableLoader
 from core.workflow.workflow_entry import WorkflowEntry
 from extensions.ext_database import db
-from models import Workflow
 from models.enums import UserFrom
-from models.model import App, Conversation, Message, MessageAnnotation
+from models.model import App, Conversation, EndUser, Message
 from models.workflow import ConversationVariable, WorkflowType
 
 logger = logging.getLogger(__name__)
@@ -44,38 +36,42 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
 
     def __init__(
         self,
-        *,
         application_generate_entity: AdvancedChatAppGenerateEntity,
         queue_manager: AppQueueManager,
         conversation: Conversation,
         message: Message,
         dialogue_count: int,
         variable_loader: VariableLoader,
-        workflow: Workflow,
-        system_user_id: str,
-        app: App,
     ) -> None:
-        super().__init__(
+        super().__init__(queue_manager, variable_loader)
-            queue_manager=queue_manager,
-            variable_loader=variable_loader,
-            app_id=application_generate_entity.app_config.app_id,
-        )
         self.application_generate_entity = application_generate_entity
         self.conversation = conversation
         self.message = message
         self._dialogue_count = dialogue_count
-        self._workflow = workflow
+
-        self.system_user_id = system_user_id
+    def _get_app_id(self) -> str:
-        self._app = app
+        return self.application_generate_entity.app_config.app_id
 
     def run(self) -> None:
         app_config = self.application_generate_entity.app_config
         app_config = cast(AdvancedChatAppConfig, app_config)
 
-        app_record = db.session.query(App).where(App.id == app_config.app_id).first()
+        app_record = db.session.query(App).filter(App.id == app_config.app_id).first()
         if not app_record:
             raise ValueError("App not found")
 
+        workflow = self.get_workflow(app_model=app_record, workflow_id=app_config.workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not initialized")
+
+        user_id = None
+        if self.application_generate_entity.invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}:
+            end_user = db.session.query(EndUser).filter(EndUser.id == self.application_generate_entity.user_id).first()
+            if end_user:
+                user_id = end_user.session_id
+        else:
+            user_id = self.application_generate_entity.user_id
+
         workflow_callbacks: list[WorkflowCallback] = []
         if dify_config.DEBUG:
             workflow_callbacks.append(WorkflowLoggingCallback())
@@ -83,14 +79,14 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
         if self.application_generate_entity.single_iteration_run:
             # if only single iteration run is requested
             graph, variable_pool = self._get_graph_and_variable_pool_of_single_iteration(
-                workflow=self._workflow,
+                workflow=workflow,
                 node_id=self.application_generate_entity.single_iteration_run.node_id,
                 user_inputs=dict(self.application_generate_entity.single_iteration_run.inputs),
             )
         elif self.application_generate_entity.single_loop_run:
             # if only single loop run is requested
             graph, variable_pool = self._get_graph_and_variable_pool_of_single_loop(
-                workflow=self._workflow,
+                workflow=workflow,
                 node_id=self.application_generate_entity.single_loop_run.node_id,
                 user_inputs=dict(self.application_generate_entity.single_loop_run.inputs),
             )
@@ -101,7 +97,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
 
             # moderation
             if self.handle_input_moderation(
-                app_record=self._app,
+                app_record=app_record,
                 app_generate_entity=self.application_generate_entity,
                 inputs=inputs,
                 query=query,
@@ -111,7 +107,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
 
             # annotation reply
             if self.handle_annotation_reply(
-                app_record=self._app,
+                app_record=app_record,
                 message=self.message,
                 query=query,
                 app_generate_entity=self.application_generate_entity,
@@ -131,7 +127,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
                         ConversationVariable.from_variable(
                             app_id=self.conversation.app_id, conversation_id=self.conversation.id, variable=variable
                         )
-                        for variable in self._workflow.conversation_variables
+                        for variable in workflow.conversation_variables
                     ]
                     session.add_all(db_conversation_variables)
                 # Convert database entities to variables.
@@ -140,40 +136,38 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
                 session.commit()
 
             # Create a variable pool.
-            system_inputs = SystemVariable(
+            system_inputs = {
-                query=query,
+                SystemVariableKey.QUERY: query,
-                files=files,
+                SystemVariableKey.FILES: files,
-                conversation_id=self.conversation.id,
+                SystemVariableKey.CONVERSATION_ID: self.conversation.id,
-                user_id=self.system_user_id,
+                SystemVariableKey.USER_ID: user_id,
-                dialogue_count=self._dialogue_count,
+                SystemVariableKey.DIALOGUE_COUNT: self._dialogue_count,
-                app_id=app_config.app_id,
+                SystemVariableKey.APP_ID: app_config.app_id,
-                workflow_id=app_config.workflow_id,
+                SystemVariableKey.WORKFLOW_ID: app_config.workflow_id,
-                workflow_execution_id=self.application_generate_entity.workflow_run_id,
+                SystemVariableKey.WORKFLOW_EXECUTION_ID: self.application_generate_entity.workflow_run_id,
-            )
+            }
 
             # init variable pool
             variable_pool = VariablePool(
                 system_variables=system_inputs,
                 user_inputs=inputs,
-                environment_variables=self._workflow.environment_variables,
+                environment_variables=workflow.environment_variables,
-                # Based on the definition of `VariableUnion`,
+                conversation_variables=conversation_variables,
-                # `list[Variable]` can be safely used as `list[VariableUnion]` since they are compatible.
-                conversation_variables=cast(list[VariableUnion], conversation_variables),
             )
 
             # init graph
-            graph = self._init_graph(graph_config=self._workflow.graph_dict)
+            graph = self._init_graph(graph_config=workflow.graph_dict)
 
         db.session.close()
 
         # RUN WORKFLOW
         workflow_entry = WorkflowEntry(
-            tenant_id=self._workflow.tenant_id,
+            tenant_id=workflow.tenant_id,
-            app_id=self._workflow.app_id,
+            app_id=workflow.app_id,
-            workflow_id=self._workflow.id,
+            workflow_id=workflow.id,
-            workflow_type=WorkflowType.value_of(self._workflow.type),
+            workflow_type=WorkflowType.value_of(workflow.type),
             graph=graph,
-            graph_config=self._workflow.graph_dict,
+            graph_config=workflow.graph_dict,
             user_id=self.application_generate_entity.user_id,
             user_from=(
                 UserFrom.ACCOUNT
@@ -244,51 +238,3 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
         self._publish_event(QueueTextChunkEvent(text=text))
 
         self._publish_event(QueueStopEvent(stopped_by=stopped_by))
-
-    def query_app_annotations_to_reply(
-        self, app_record: App, message: Message, query: str, user_id: str, invoke_from: InvokeFrom
-    ) -> Optional[MessageAnnotation]:
-        """
-        Query app annotations to reply
-        :param app_record: app record
-        :param message: message
-        :param query: query
-        :param user_id: user id
-        :param invoke_from: invoke from
-        :return:
-        """
-        annotation_reply_feature = AnnotationReplyFeature()
-        return annotation_reply_feature.query(
-            app_record=app_record, message=message, query=query, user_id=user_id, invoke_from=invoke_from
-        )
-
-    def moderation_for_inputs(
-        self,
-        *,
-        app_id: str,
-        tenant_id: str,
-        app_generate_entity: AppGenerateEntity,
-        inputs: Mapping[str, Any],
-        query: str | None = None,
-        message_id: str,
-    ) -> tuple[bool, Mapping[str, Any], str]:
-        """
-        Process sensitive_word_avoidance.
-        :param app_id: app id
-        :param tenant_id: tenant id
-        :param app_generate_entity: app generate entity
-        :param inputs: inputs
-        :param query: query
-        :param message_id: message id
-        :return:
-        """
-        moderation_feature = InputModeration()
-        return moderation_feature.check(
-            app_id=app_id,
-            tenant_id=tenant_id,
-            app_config=app_generate_entity.app_config,
-            inputs=dict(inputs),
-            query=query or "",
-            message_id=message_id,
-            trace_manager=app_generate_entity.trace_manager,
-        )

api/core/app/apps/agent_chat/app_generator.py

@@ -15,8 +15,7 @@ from core.app.app_config.features.file_upload.manager import FileUploadConfigMan
 from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager
 from core.app.apps.agent_chat.app_runner import AgentChatAppRunner
 from core.app.apps.agent_chat.generate_response_converter import AgentChatAppGenerateResponseConverter
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.apps.message_based_app_generator import MessageBasedAppGenerator
 from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager
 from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, InvokeFrom

api/core/app/apps/agent_chat/app_runner.py

@@ -45,7 +45,7 @@ class AgentChatAppRunner(AppRunner):
         app_config = application_generate_entity.app_config
         app_config = cast(AgentChatAppConfig, app_config)
 
-        app_record = db.session.query(App).where(App.id == app_config.app_id).first()
+        app_record = db.session.query(App).filter(App.id == app_config.app_id).first()
         if not app_record:
             raise ValueError("App not found")
 
@@ -183,10 +183,10 @@ class AgentChatAppRunner(AppRunner):
         if {ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL}.intersection(model_schema.features or []):
             agent_entity.strategy = AgentEntity.Strategy.FUNCTION_CALLING
 
-        conversation_result = db.session.query(Conversation).where(Conversation.id == conversation.id).first()
+        conversation_result = db.session.query(Conversation).filter(Conversation.id == conversation.id).first()
         if conversation_result is None:
             raise ValueError("Conversation not found")
-        message_result = db.session.query(Message).where(Message.id == message.id).first()
+        message_result = db.session.query(Message).filter(Message.id == message.id).first()
         if message_result is None:
             raise ValueError("Message not found")
         db.session.close()

api/core/app/apps/base_app_queue_manager.py

@@ -169,3 +169,7 @@ class AppQueueManager:
                 raise TypeError(
                     "Critical Error: Passing SQLAlchemy Model instances that cause thread safety issues is not allowed."
                 )
+
+
+class GenerateTaskStoppedError(Exception):
+    pass

api/core/app/apps/base_app_runner.py

@@ -38,6 +38,69 @@ _logger = logging.getLogger(__name__)
 
 
 class AppRunner:
+    def get_pre_calculate_rest_tokens(
+        self,
+        app_record: App,
+        model_config: ModelConfigWithCredentialsEntity,
+        prompt_template_entity: PromptTemplateEntity,
+        inputs: Mapping[str, str],
+        files: Sequence["File"],
+        query: Optional[str] = None,
+    ) -> int:
+        """
+        Get pre calculate rest tokens
+        :param app_record: app record
+        :param model_config: model config entity
+        :param prompt_template_entity: prompt template entity
+        :param inputs: inputs
+        :param files: files
+        :param query: query
+        :return:
+        """
+        # Invoke model
+        model_instance = ModelInstance(
+            provider_model_bundle=model_config.provider_model_bundle, model=model_config.model
+        )
+
+        model_context_tokens = model_config.model_schema.model_properties.get(ModelPropertyKey.CONTEXT_SIZE)
+
+        max_tokens = 0
+        for parameter_rule in model_config.model_schema.parameter_rules:
+            if parameter_rule.name == "max_tokens" or (
+                parameter_rule.use_template and parameter_rule.use_template == "max_tokens"
+            ):
+                max_tokens = (
+                    model_config.parameters.get(parameter_rule.name)
+                    or model_config.parameters.get(parameter_rule.use_template or "")
+                ) or 0
+
+        if model_context_tokens is None:
+            return -1
+
+        if max_tokens is None:
+            max_tokens = 0
+
+        # get prompt messages without memory and context
+        prompt_messages, stop = self.organize_prompt_messages(
+            app_record=app_record,
+            model_config=model_config,
+            prompt_template_entity=prompt_template_entity,
+            inputs=inputs,
+            files=files,
+            query=query,
+        )
+
+        prompt_tokens = model_instance.get_llm_num_tokens(prompt_messages)
+
+        rest_tokens: int = model_context_tokens - max_tokens - prompt_tokens
+        if rest_tokens < 0:
+            raise InvokeBadRequestError(
+                "Query or prefix prompt is too long, you can reduce the prefix prompt, "
+                "or shrink the max token, or switch to a llm with a larger token limit size."
+            )
+
+        return rest_tokens
+
     def recalc_llm_max_tokens(
         self, model_config: ModelConfigWithCredentialsEntity, prompt_messages: list[PromptMessage]
     ):
@@ -118,7 +181,7 @@ class AppRunner:
         else:
             memory_config = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=False))
 
-            model_mode = ModelMode(model_config.mode)
+            model_mode = ModelMode.value_of(model_config.mode)
             prompt_template: Union[CompletionModelPromptTemplate, list[ChatModelMessage]]
             if model_mode == ModelMode.COMPLETION:
                 advanced_completion_prompt_template = prompt_template_entity.advanced_completion_prompt_template

api/core/app/apps/chat/app_generator.py

@@ -11,11 +11,10 @@ from configs import dify_config
 from constants import UUID_NIL
 from core.app.app_config.easy_ui_based_app.model_config.converter import ModelConfigConverter
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
 from core.app.apps.chat.app_config_manager import ChatAppConfigManager
 from core.app.apps.chat.app_runner import ChatAppRunner
 from core.app.apps.chat.generate_response_converter import ChatAppGenerateResponseConverter
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.apps.message_based_app_generator import MessageBasedAppGenerator
 from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager
 from core.app.entities.app_invoke_entities import ChatAppGenerateEntity, InvokeFrom

api/core/app/apps/chat/app_runner.py

@@ -43,7 +43,7 @@ class ChatAppRunner(AppRunner):
         app_config = application_generate_entity.app_config
         app_config = cast(ChatAppConfig, app_config)
 
-        app_record = db.session.query(App).where(App.id == app_config.app_id).first()
+        app_record = db.session.query(App).filter(App.id == app_config.app_id).first()
         if not app_record:
             raise ValueError("App not found")
 

api/core/app/apps/completion/app_generator.py

@@ -10,11 +10,10 @@ from pydantic import ValidationError
 from configs import dify_config
 from core.app.app_config.easy_ui_based_app.model_config.converter import ModelConfigConverter
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
 from core.app.apps.completion.app_config_manager import CompletionAppConfigManager
 from core.app.apps.completion.app_runner import CompletionAppRunner
 from core.app.apps.completion.generate_response_converter import CompletionAppGenerateResponseConverter
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.apps.message_based_app_generator import MessageBasedAppGenerator
 from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager
 from core.app.entities.app_invoke_entities import CompletionAppGenerateEntity, InvokeFrom
@@ -248,7 +247,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         """
         message = (
             db.session.query(Message)
-            .where(
+            .filter(
                 Message.id == message_id,
                 Message.app_id == app_model.id,
                 Message.from_source == ("api" if isinstance(user, EndUser) else "console"),

api/core/app/apps/completion/app_runner.py

@@ -36,7 +36,7 @@ class CompletionAppRunner(AppRunner):
         app_config = application_generate_entity.app_config
         app_config = cast(CompletionAppConfig, app_config)
 
-        app_record = db.session.query(App).where(App.id == app_config.app_id).first()
+        app_record = db.session.query(App).filter(App.id == app_config.app_id).first()
         if not app_record:
             raise ValueError("App not found")
 

api/core/app/apps/exc.py

@@ -1,2 +0,0 @@
-class GenerateTaskStoppedError(Exception):
-    pass

api/core/app/apps/message_based_app_generator.py

@@ -1,12 +1,12 @@
 import json
 import logging
 from collections.abc import Generator
+from datetime import UTC, datetime
 from typing import Optional, Union, cast
 
 from core.app.app_config.entities import EasyUIBasedAppConfig, EasyUIBasedAppModelConfigFrom
 from core.app.apps.base_app_generator import BaseAppGenerator
-from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.entities.app_invoke_entities import (
     AdvancedChatAppGenerateEntity,
     AgentChatAppGenerateEntity,
@@ -24,7 +24,6 @@ from core.app.entities.task_entities import (
 from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
 from extensions.ext_database import db
-from libs.datetime_utils import naive_utc_now
 from models import Account
 from models.enums import CreatorUserRole
 from models.model import App, AppMode, AppModelConfig, Conversation, EndUser, Message, MessageFile
@@ -85,7 +84,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
         if conversation:
             app_model_config = (
                 db.session.query(AppModelConfig)
-                .where(AppModelConfig.id == conversation.app_model_config_id, AppModelConfig.app_id == app_model.id)
+                .filter(AppModelConfig.id == conversation.app_model_config_id, AppModelConfig.app_id == app_model.id)
                 .first()
             )
 
@@ -151,7 +150,13 @@ class MessageBasedAppGenerator(BaseAppGenerator):
         introduction = self._get_conversation_introduction(application_generate_entity)
 
         # get conversation name
-        query = application_generate_entity.query or "New conversation"
+        if isinstance(application_generate_entity, AdvancedChatAppGenerateEntity):
+            query = application_generate_entity.query or "New conversation"
+        else:
+            query = next(iter(application_generate_entity.inputs.values()), "New conversation")
+            if isinstance(query, int):
+                query = str(query)
+        query = query or "New conversation"
         conversation_name = (query[:20] + "…") if len(query) > 20 else query
 
         if not conversation:
@@ -178,7 +183,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             db.session.commit()
             db.session.refresh(conversation)
         else:
-            conversation.updated_at = naive_utc_now()
+            conversation.updated_at = datetime.now(UTC).replace(tzinfo=None)
             db.session.commit()
 
         message = Message(
@@ -253,7 +258,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
         :param conversation_id: conversation id
         :return: conversation
         """
-        conversation = db.session.query(Conversation).where(Conversation.id == conversation_id).first()
+        conversation = db.session.query(Conversation).filter(Conversation.id == conversation_id).first()
 
         if not conversation:
             raise ConversationNotExistsError("Conversation not exists")
@@ -266,7 +271,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
         :param message_id: message id
         :return: message
         """
-        message = db.session.query(Message).where(Message.id == message_id).first()
+        message = db.session.query(Message).filter(Message.id == message_id).first()
 
         if message is None:
             raise MessageNotExistsError("Message not exists")

api/core/app/apps/message_based_app_queue_manager.py

@@ -1,5 +1,4 @@
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.app.entities.queue_entities import (
     AppQueueEvent,

api/core/app/apps/workflow/app_generator.py

@@ -7,15 +7,13 @@ from typing import Any, Literal, Optional, Union, overload
 
 from flask import Flask, current_app
 from pydantic import ValidationError
-from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
-from sqlalchemy.orm import Session, sessionmaker
 
 import contexts
 from configs import dify_config
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.app.apps.base_app_generator import BaseAppGenerator
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
 from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager
 from core.app.apps.workflow.app_runner import WorkflowAppRunner
@@ -23,10 +21,10 @@ from core.app.apps.workflow.generate_response_converter import WorkflowAppGenera
 from core.app.apps.workflow.generate_task_pipeline import WorkflowAppGenerateTaskPipeline
 from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity
 from core.app.entities.task_entities import WorkflowAppBlockingResponse, WorkflowAppStreamResponse
-from core.helper.trace_id_helper import extract_external_trace_id_from_args
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
 from core.ops.ops_trace_manager import TraceQueueManager
-from core.repositories import DifyCoreRepositoryFactory
+from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository
+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository
 from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory
 from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository
 from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository
@@ -125,10 +123,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
         )
 
         inputs: Mapping[str, Any] = args["inputs"]
-
-        extras = {
-            **extract_external_trace_id_from_args(args),
-        }
         workflow_run_id = str(uuid.uuid4())
         # init application generate entity
         application_generate_entity = WorkflowAppGenerateEntity(
@@ -148,7 +142,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
             call_depth=call_depth,
             trace_manager=trace_manager,
             workflow_execution_id=workflow_run_id,
-            extras=extras,
         )
 
         contexts.plugin_tool_providers.set({})
@@ -163,14 +156,14 @@ class WorkflowAppGenerator(BaseAppGenerator):
             workflow_triggered_from = WorkflowRunTriggeredFrom.DEBUGGING
         else:
             workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN
-        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
+        workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
             triggered_from=workflow_triggered_from,
         )
         # Create workflow node execution repository
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
+        workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
@@ -313,14 +306,16 @@ class WorkflowAppGenerator(BaseAppGenerator):
         # Create session factory
         session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
         # Create workflow execution(aka workflow run) repository
-        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
+        workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
             triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
         )
         # Create workflow node execution repository
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
+        session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
+
+        workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
@@ -395,14 +390,16 @@ class WorkflowAppGenerator(BaseAppGenerator):
         # Create session factory
         session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
         # Create workflow execution(aka workflow run) repository
-        workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
+        workflow_execution_repository = SQLAlchemyWorkflowExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
             triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
         )
         # Create workflow node execution repository
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
+        session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
+
+        workflow_node_execution_repository = SQLAlchemyWorkflowNodeExecutionRepository(
             session_factory=session_factory,
             user=user,
             app_id=application_generate_entity.app_config.app_id,
@@ -446,44 +443,17 @@ class WorkflowAppGenerator(BaseAppGenerator):
         """
 
         with preserve_flask_contexts(flask_app, context_vars=context):
-            with Session(db.engine, expire_on_commit=False) as session:
+            try:
-                workflow = session.scalar(
+                # workflow app
-                    select(Workflow).where(
+                runner = WorkflowAppRunner(
-                        Workflow.tenant_id == application_generate_entity.app_config.tenant_id,
+                    application_generate_entity=application_generate_entity,
-                        Workflow.app_id == application_generate_entity.app_config.app_id,
+                    queue_manager=queue_manager,
-                        Workflow.id == application_generate_entity.app_config.workflow_id,
+                    workflow_thread_pool_id=workflow_thread_pool_id,
-                    )
+                    variable_loader=variable_loader,
                 )
-                if workflow is None:
-                    raise ValueError("Workflow not found")
-
-                # Determine system_user_id based on invocation source
-                is_external_api_call = application_generate_entity.invoke_from in {
-                    InvokeFrom.WEB_APP,
-                    InvokeFrom.SERVICE_API,
-                }
-
-                if is_external_api_call:
-                    # For external API calls, use end user's session ID
-                    end_user = session.scalar(select(EndUser).where(EndUser.id == application_generate_entity.user_id))
-                    system_user_id = end_user.session_id if end_user else ""
-                else:
-                    # For internal calls, use the original user ID
-                    system_user_id = application_generate_entity.user_id
-
-            runner = WorkflowAppRunner(
-                application_generate_entity=application_generate_entity,
-                queue_manager=queue_manager,
-                workflow_thread_pool_id=workflow_thread_pool_id,
-                variable_loader=variable_loader,
-                workflow=workflow,
-                system_user_id=system_user_id,
-            )
 
-            try:
                 runner.run()
-            except GenerateTaskStoppedError as e:
+            except GenerateTaskStoppedError:
-                logger.warning(f"Task stopped: {str(e)}")
                 pass
             except InvokeAuthorizationError:
                 queue_manager.publish_error(
@@ -499,6 +469,8 @@ class WorkflowAppGenerator(BaseAppGenerator):
             except Exception as e:
                 logger.exception("Unknown Error when generating")
                 queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
+            finally:
+                db.session.close()
 
     def _handle_response(
         self,

api/core/app/apps/workflow/app_queue_manager.py

@@ -1,5 +1,4 @@
-from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedError, PublishFrom
-from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.app.entities.queue_entities import (
     AppQueueEvent,

api/core/app/apps/workflow/app_runner.py

@@ -11,11 +11,13 @@ from core.app.entities.app_invoke_entities import (
 )
 from core.workflow.callbacks import WorkflowCallback, WorkflowLoggingCallback
 from core.workflow.entities.variable_pool import VariablePool
-from core.workflow.system_variable import SystemVariable
+from core.workflow.enums import SystemVariableKey
 from core.workflow.variable_loader import VariableLoader
 from core.workflow.workflow_entry import WorkflowEntry
+from extensions.ext_database import db
 from models.enums import UserFrom
-from models.workflow import Workflow, WorkflowType
+from models.model import App, EndUser
+from models.workflow import WorkflowType
 
 logger = logging.getLogger(__name__)
 
@@ -27,23 +29,22 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
 
     def __init__(
         self,
-        *,
         application_generate_entity: WorkflowAppGenerateEntity,
         queue_manager: AppQueueManager,
         variable_loader: VariableLoader,
         workflow_thread_pool_id: Optional[str] = None,
-        workflow: Workflow,
-        system_user_id: str,
     ) -> None:
-        super().__init__(
+        """
-            queue_manager=queue_manager,
+        :param application_generate_entity: application generate entity
-            variable_loader=variable_loader,
+        :param queue_manager: application queue manager
-            app_id=application_generate_entity.app_config.app_id,
+        :param workflow_thread_pool_id: workflow thread pool id
-        )
+        """
+        super().__init__(queue_manager, variable_loader)
         self.application_generate_entity = application_generate_entity
         self.workflow_thread_pool_id = workflow_thread_pool_id
-        self._workflow = workflow
+
-        self._sys_user_id = system_user_id
+    def _get_app_id(self) -> str:
+        return self.application_generate_entity.app_config.app_id
 
     def run(self) -> None:
         """
@@ -52,6 +53,24 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
         app_config = self.application_generate_entity.app_config
         app_config = cast(WorkflowAppConfig, app_config)
 
+        user_id = None
+        if self.application_generate_entity.invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}:
+            end_user = db.session.query(EndUser).filter(EndUser.id == self.application_generate_entity.user_id).first()
+            if end_user:
+                user_id = end_user.session_id
+        else:
+            user_id = self.application_generate_entity.user_id
+
+        app_record = db.session.query(App).filter(App.id == app_config.app_id).first()
+        if not app_record:
+            raise ValueError("App not found")
+
+        workflow = self.get_workflow(app_model=app_record, workflow_id=app_config.workflow_id)
+        if not workflow:
+            raise ValueError("Workflow not initialized")
+
+        db.session.close()
+
         workflow_callbacks: list[WorkflowCallback] = []
         if dify_config.DEBUG:
             workflow_callbacks.append(WorkflowLoggingCallback())
@@ -60,14 +79,14 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
         if self.application_generate_entity.single_iteration_run:
             # if only single iteration run is requested
             graph, variable_pool = self._get_graph_and_variable_pool_of_single_iteration(
-                workflow=self._workflow,
+                workflow=workflow,
                 node_id=self.application_generate_entity.single_iteration_run.node_id,
                 user_inputs=self.application_generate_entity.single_iteration_run.inputs,
             )
         elif self.application_generate_entity.single_loop_run:
             # if only single loop run is requested
             graph, variable_pool = self._get_graph_and_variable_pool_of_single_loop(
-                workflow=self._workflow,
+                workflow=workflow,
                 node_id=self.application_generate_entity.single_loop_run.node_id,
                 user_inputs=self.application_generate_entity.single_loop_run.inputs,
             )
@@ -76,33 +95,32 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
             files = self.application_generate_entity.files
 
             # Create a variable pool.
-
+            system_inputs = {
-            system_inputs = SystemVariable(
+                SystemVariableKey.FILES: files,
-                files=files,
+                SystemVariableKey.USER_ID: user_id,
-                user_id=self._sys_user_id,
+                SystemVariableKey.APP_ID: app_config.app_id,
-                app_id=app_config.app_id,
+                SystemVariableKey.WORKFLOW_ID: app_config.workflow_id,
-                workflow_id=app_config.workflow_id,
+                SystemVariableKey.WORKFLOW_EXECUTION_ID: self.application_generate_entity.workflow_execution_id,
-                workflow_execution_id=self.application_generate_entity.workflow_execution_id,
+            }
-            )
 
             variable_pool = VariablePool(
                 system_variables=system_inputs,
                 user_inputs=inputs,
-                environment_variables=self._workflow.environment_variables,
+                environment_variables=workflow.environment_variables,
                 conversation_variables=[],
             )
 
             # init graph
-            graph = self._init_graph(graph_config=self._workflow.graph_dict)
+            graph = self._init_graph(graph_config=workflow.graph_dict)
 
         # RUN WORKFLOW
         workflow_entry = WorkflowEntry(
-            tenant_id=self._workflow.tenant_id,
+            tenant_id=workflow.tenant_id,
-            app_id=self._workflow.app_id,
+            app_id=workflow.app_id,
-            workflow_id=self._workflow.id,
+            workflow_id=workflow.id,
-            workflow_type=WorkflowType.value_of(self._workflow.type),
+            workflow_type=WorkflowType.value_of(workflow.type),
             graph=graph,
-            graph_config=self._workflow.graph_dict,
+            graph_config=workflow.graph_dict,
             user_id=self.application_generate_entity.user_id,
             user_from=(
                 UserFrom.ACCOUNT

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -1,9 +1,9 @@
 import logging
 import time
-from collections.abc import Callable, Generator
+from collections.abc import Generator
-from contextlib import contextmanager
+from typing import Optional, Union
-from typing import Any, Optional, Union
 
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from constants.tts_auto_play_timeout import TTS_AUTO_PLAY_TIMEOUT, TTS_AUTO_PLAY_YIELD_CPU_TIME
@@ -14,7 +14,6 @@ from core.app.entities.app_invoke_entities import (
     WorkflowAppGenerateEntity,
 )
 from core.app.entities.queue_entities import (
-    MessageQueueMessage,
     QueueAgentLogEvent,
     QueueErrorEvent,
     QueueIterationCompletedEvent,
@@ -40,13 +39,11 @@ from core.app.entities.queue_entities import (
     QueueWorkflowPartialSuccessEvent,
     QueueWorkflowStartedEvent,
     QueueWorkflowSucceededEvent,
-    WorkflowQueueMessage,
 )
 from core.app.entities.task_entities import (
     ErrorStreamResponse,
     MessageAudioEndStreamResponse,
     MessageAudioStreamResponse,
-    PingStreamResponse,
     StreamResponse,
     TextChunkStreamResponse,
     WorkflowAppBlockingResponse,
@@ -58,11 +55,10 @@ from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTas
 from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk
 from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.entities.workflow_execution import WorkflowExecution, WorkflowExecutionStatus, WorkflowType
-from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
+from core.workflow.enums import SystemVariableKey
 from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory
 from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository
 from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository
-from core.workflow.system_variable import SystemVariable
 from core.workflow.workflow_cycle_manager import CycleManagerWorkflowInfo, WorkflowCycleManager
 from extensions.ext_database import db
 from models.account import Account
@@ -72,6 +68,7 @@ from models.workflow import (
     Workflow,
     WorkflowAppLog,
     WorkflowAppLogCreatedFrom,
+    WorkflowRun,
 )
 
 logger = logging.getLogger(__name__)
@@ -112,13 +109,13 @@ class WorkflowAppGenerateTaskPipeline:
 
         self._workflow_cycle_manager = WorkflowCycleManager(
             application_generate_entity=application_generate_entity,
-            workflow_system_variables=SystemVariable(
+            workflow_system_variables={
-                files=application_generate_entity.files,
+                SystemVariableKey.FILES: application_generate_entity.files,
-                user_id=user_session_id,
+                SystemVariableKey.USER_ID: user_session_id,
-                app_id=application_generate_entity.app_config.app_id,
+                SystemVariableKey.APP_ID: application_generate_entity.app_config.app_id,
-                workflow_id=workflow.id,
+                SystemVariableKey.WORKFLOW_ID: workflow.id,
-                workflow_execution_id=application_generate_entity.workflow_execution_id,
+                SystemVariableKey.WORKFLOW_EXECUTION_ID: application_generate_entity.workflow_execution_id,
-            ),
+            },
             workflow_info=CycleManagerWorkflowInfo(
                 workflow_id=workflow.id,
                 workflow_type=WorkflowType(workflow.type),
@@ -251,500 +248,322 @@ class WorkflowAppGenerateTaskPipeline:
         if tts_publisher:
             yield MessageAudioEndStreamResponse(audio="", task_id=task_id)
 
-    @contextmanager
+    def _process_stream_response(
-    def _database_session(self):
-        """Context manager for database sessions."""
-        with Session(db.engine, expire_on_commit=False) as session:
-            try:
-                yield session
-                session.commit()
-            except Exception:
-                session.rollback()
-                raise
-
-    def _ensure_workflow_initialized(self) -> None:
-        """Fluent validation for workflow state."""
-        if not self._workflow_run_id:
-            raise ValueError("workflow run not initialized.")
-
-    def _ensure_graph_runtime_initialized(self, graph_runtime_state: Optional[GraphRuntimeState]) -> GraphRuntimeState:
-        """Fluent validation for graph runtime state."""
-        if not graph_runtime_state:
-            raise ValueError("graph runtime state not initialized.")
-        return graph_runtime_state
-
-    def _handle_ping_event(self, event: QueuePingEvent, **kwargs) -> Generator[PingStreamResponse, None, None]:
-        """Handle ping events."""
-        yield self._base_task_pipeline._ping_stream_response()
-
-    def _handle_error_event(self, event: QueueErrorEvent, **kwargs) -> Generator[ErrorStreamResponse, None, None]:
-        """Handle error events."""
-        err = self._base_task_pipeline._handle_error(event=event)
-        yield self._base_task_pipeline._error_to_stream_response(err)
-
-    def _handle_workflow_started_event(
-        self, event: QueueWorkflowStartedEvent, **kwargs
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle workflow started events."""
-        # init workflow run
-        workflow_execution = self._workflow_cycle_manager.handle_workflow_run_start()
-        self._workflow_run_id = workflow_execution.id_
-        start_resp = self._workflow_response_converter.workflow_start_to_stream_response(
-            task_id=self._application_generate_entity.task_id,
-            workflow_execution=workflow_execution,
-        )
-        yield start_resp
-
-    def _handle_node_retry_event(self, event: QueueNodeRetryEvent, **kwargs) -> Generator[StreamResponse, None, None]:
-        """Handle node retry events."""
-        self._ensure_workflow_initialized()
-
-        with self._database_session() as session:
-            workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_retried(
-                workflow_execution_id=self._workflow_run_id,
-                event=event,
-            )
-            response = self._workflow_response_converter.workflow_node_retry_to_stream_response(
-                event=event,
-                task_id=self._application_generate_entity.task_id,
-                workflow_node_execution=workflow_node_execution,
-            )
-
-        if response:
-            yield response
-
-    def _handle_node_started_event(
-        self, event: QueueNodeStartedEvent, **kwargs
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle node started events."""
-        self._ensure_workflow_initialized()
-
-        workflow_node_execution = self._workflow_cycle_manager.handle_node_execution_start(
-            workflow_execution_id=self._workflow_run_id, event=event
-        )
-        node_start_response = self._workflow_response_converter.workflow_node_start_to_stream_response(
-            event=event,
-            task_id=self._application_generate_entity.task_id,
-            workflow_node_execution=workflow_node_execution,
-        )
-
-        if node_start_response:
-            yield node_start_response
-
-    def _handle_node_succeeded_event(
-        self, event: QueueNodeSucceededEvent, **kwargs
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle node succeeded events."""
-        workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_success(event=event)
-        node_success_response = self._workflow_response_converter.workflow_node_finish_to_stream_response(
-            event=event,
-            task_id=self._application_generate_entity.task_id,
-            workflow_node_execution=workflow_node_execution,
-        )
-
-        self._save_output_for_event(event, workflow_node_execution.id)
-
-        if node_success_response:
-            yield node_success_response
-
-    def _handle_node_failed_events(
         self,
-        event: Union[
+        tts_publisher: Optional[AppGeneratorTTSPublisher] = None,
-            QueueNodeFailedEvent, QueueNodeInIterationFailedEvent, QueueNodeInLoopFailedEvent, QueueNodeExceptionEvent
+        trace_manager: Optional[TraceQueueManager] = None,
-        ],
-        **kwargs,
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle various node failure events."""
-        workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_failed(
-            event=event,
-        )
-        node_failed_response = self._workflow_response_converter.workflow_node_finish_to_stream_response(
-            event=event,
-            task_id=self._application_generate_entity.task_id,
-            workflow_node_execution=workflow_node_execution,
-        )
-
-        if isinstance(event, QueueNodeExceptionEvent):
-            self._save_output_for_event(event, workflow_node_execution.id)
-
-        if node_failed_response:
-            yield node_failed_response
-
-    def _handle_parallel_branch_started_event(
-        self, event: QueueParallelBranchRunStartedEvent, **kwargs
     ) -> Generator[StreamResponse, None, None]:
-        """Handle parallel branch started events."""
+        """
-        self._ensure_workflow_initialized()
+        Process stream response.
-
+        :return:
-        parallel_start_resp = self._workflow_response_converter.workflow_parallel_branch_start_to_stream_response(
+        """
-            task_id=self._application_generate_entity.task_id,
+        graph_runtime_state = None
-            workflow_execution_id=self._workflow_run_id,
-            event=event,
-        )
-        yield parallel_start_resp
 
-    def _handle_parallel_branch_finished_events(
+        for queue_message in self._base_task_pipeline._queue_manager.listen():
-        self, event: Union[QueueParallelBranchRunSucceededEvent, QueueParallelBranchRunFailedEvent], **kwargs
+            event = queue_message.event
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle parallel branch finished events."""
-        self._ensure_workflow_initialized()
 
-        parallel_finish_resp = self._workflow_response_converter.workflow_parallel_branch_finished_to_stream_response(
+            if isinstance(event, QueuePingEvent):
-            task_id=self._application_generate_entity.task_id,
+                yield self._base_task_pipeline._ping_stream_response()
-            workflow_execution_id=self._workflow_run_id,
+            elif isinstance(event, QueueErrorEvent):
-            event=event,
+                err = self._base_task_pipeline._handle_error(event=event)
-        )
+                yield self._base_task_pipeline._error_to_stream_response(err)
-        yield parallel_finish_resp
+                break
+            elif isinstance(event, QueueWorkflowStartedEvent):
+                # override graph runtime state
+                graph_runtime_state = event.graph_runtime_state
+
+                # init workflow run
+                workflow_execution = self._workflow_cycle_manager.handle_workflow_run_start()
+                self._workflow_run_id = workflow_execution.id_
+                start_resp = self._workflow_response_converter.workflow_start_to_stream_response(
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_execution=workflow_execution,
+                )
 
-    def _handle_iteration_start_event(
+                yield start_resp
-        self, event: QueueIterationStartEvent, **kwargs
+            elif isinstance(
-    ) -> Generator[StreamResponse, None, None]:
+                event,
-        """Handle iteration start events."""
+                QueueNodeRetryEvent,
-        self._ensure_workflow_initialized()
+            ):
+                if not self._workflow_run_id:
+                    raise ValueError("workflow run not initialized.")
+                with Session(db.engine, expire_on_commit=False) as session:
+                    workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_retried(
+                        workflow_execution_id=self._workflow_run_id,
+                        event=event,
+                    )
+                    response = self._workflow_response_converter.workflow_node_retry_to_stream_response(
+                        event=event,
+                        task_id=self._application_generate_entity.task_id,
+                        workflow_node_execution=workflow_node_execution,
+                    )
+                    session.commit()
 
-        iter_start_resp = self._workflow_response_converter.workflow_iteration_start_to_stream_response(
+                if response:
-            task_id=self._application_generate_entity.task_id,
+                    yield response
-            workflow_execution_id=self._workflow_run_id,
+            elif isinstance(event, QueueNodeStartedEvent):
-            event=event,
+                if not self._workflow_run_id:
-        )
+                    raise ValueError("workflow run not initialized.")
-        yield iter_start_resp
 
-    def _handle_iteration_next_event(
+                workflow_node_execution = self._workflow_cycle_manager.handle_node_execution_start(
-        self, event: QueueIterationNextEvent, **kwargs
+                    workflow_execution_id=self._workflow_run_id, event=event
-    ) -> Generator[StreamResponse, None, None]:
+                )
-        """Handle iteration next events."""
+                node_start_response = self._workflow_response_converter.workflow_node_start_to_stream_response(
-        self._ensure_workflow_initialized()
+                    event=event,
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_node_execution=workflow_node_execution,
+                )
 
-        iter_next_resp = self._workflow_response_converter.workflow_iteration_next_to_stream_response(
+                if node_start_response:
-            task_id=self._application_generate_entity.task_id,
+                    yield node_start_response
-            workflow_execution_id=self._workflow_run_id,
+            elif isinstance(event, QueueNodeSucceededEvent):
-            event=event,
+                workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_success(
-        )
+                    event=event
-        yield iter_next_resp
+                )
+                node_success_response = self._workflow_response_converter.workflow_node_finish_to_stream_response(
+                    event=event,
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_node_execution=workflow_node_execution,
+                )
 
-    def _handle_iteration_completed_event(
+                self._save_output_for_event(event, workflow_node_execution.id)
-        self, event: QueueIterationCompletedEvent, **kwargs
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle iteration completed events."""
-        self._ensure_workflow_initialized()
 
-        iter_finish_resp = self._workflow_response_converter.workflow_iteration_completed_to_stream_response(
+                if node_success_response:
-            task_id=self._application_generate_entity.task_id,
+                    yield node_success_response
-            workflow_execution_id=self._workflow_run_id,
+            elif isinstance(
-            event=event,
+                event,
-        )
+                QueueNodeFailedEvent
-        yield iter_finish_resp
+                | QueueNodeInIterationFailedEvent
+                | QueueNodeInLoopFailedEvent
+                | QueueNodeExceptionEvent,
+            ):
+                workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_failed(
+                    event=event,
+                )
+                node_failed_response = self._workflow_response_converter.workflow_node_finish_to_stream_response(
+                    event=event,
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_node_execution=workflow_node_execution,
+                )
+                if isinstance(event, QueueNodeExceptionEvent):
+                    self._save_output_for_event(event, workflow_node_execution.id)
 
-    def _handle_loop_start_event(self, event: QueueLoopStartEvent, **kwargs) -> Generator[StreamResponse, None, None]:
+                if node_failed_response:
-        """Handle loop start events."""
+                    yield node_failed_response
-        self._ensure_workflow_initialized()
 
-        loop_start_resp = self._workflow_response_converter.workflow_loop_start_to_stream_response(
+            elif isinstance(event, QueueParallelBranchRunStartedEvent):
-            task_id=self._application_generate_entity.task_id,
+                if not self._workflow_run_id:
-            workflow_execution_id=self._workflow_run_id,
+                    raise ValueError("workflow run not initialized.")
-            event=event,
-        )
-        yield loop_start_resp
 
-    def _handle_loop_next_event(self, event: QueueLoopNextEvent, **kwargs) -> Generator[StreamResponse, None, None]:
+                parallel_start_resp = (
-        """Handle loop next events."""
+                    self._workflow_response_converter.workflow_parallel_branch_start_to_stream_response(
-        self._ensure_workflow_initialized()
+                        task_id=self._application_generate_entity.task_id,
+                        workflow_execution_id=self._workflow_run_id,
+                        event=event,
+                    )
+                )
 
-        loop_next_resp = self._workflow_response_converter.workflow_loop_next_to_stream_response(
+                yield parallel_start_resp
-            task_id=self._application_generate_entity.task_id,
-            workflow_execution_id=self._workflow_run_id,
-            event=event,
-        )
-        yield loop_next_resp
 
-    def _handle_loop_completed_event(
+            elif isinstance(event, QueueParallelBranchRunSucceededEvent | QueueParallelBranchRunFailedEvent):
-        self, event: QueueLoopCompletedEvent, **kwargs
+                if not self._workflow_run_id:
-    ) -> Generator[StreamResponse, None, None]:
+                    raise ValueError("workflow run not initialized.")
-        """Handle loop completed events."""
-        self._ensure_workflow_initialized()
 
-        loop_finish_resp = self._workflow_response_converter.workflow_loop_completed_to_stream_response(
+                parallel_finish_resp = (
-            task_id=self._application_generate_entity.task_id,
+                    self._workflow_response_converter.workflow_parallel_branch_finished_to_stream_response(
-            workflow_execution_id=self._workflow_run_id,
+                        task_id=self._application_generate_entity.task_id,
-            event=event,
+                        workflow_execution_id=self._workflow_run_id,
-        )
+                        event=event,
-        yield loop_finish_resp
+                    )
-
+                )
-    def _handle_workflow_succeeded_event(
-        self,
-        event: QueueWorkflowSucceededEvent,
-        *,
-        graph_runtime_state: Optional[GraphRuntimeState] = None,
-        trace_manager: Optional[TraceQueueManager] = None,
-        **kwargs,
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle workflow succeeded events."""
-        self._ensure_workflow_initialized()
-        validated_state = self._ensure_graph_runtime_initialized(graph_runtime_state)
-
-        with self._database_session() as session:
-            workflow_execution = self._workflow_cycle_manager.handle_workflow_run_success(
-                workflow_run_id=self._workflow_run_id,
-                total_tokens=validated_state.total_tokens,
-                total_steps=validated_state.node_run_steps,
-                outputs=event.outputs,
-                conversation_id=None,
-                trace_manager=trace_manager,
-                external_trace_id=self._application_generate_entity.extras.get("external_trace_id"),
-            )
 
-            # save workflow app log
+                yield parallel_finish_resp
-            self._save_workflow_app_log(session=session, workflow_execution=workflow_execution)
 
-            workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
+            elif isinstance(event, QueueIterationStartEvent):
-                session=session,
+                if not self._workflow_run_id:
-                task_id=self._application_generate_entity.task_id,
+                    raise ValueError("workflow run not initialized.")
-                workflow_execution=workflow_execution,
-            )
 
-        yield workflow_finish_resp
+                iter_start_resp = self._workflow_response_converter.workflow_iteration_start_to_stream_response(
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_execution_id=self._workflow_run_id,
+                    event=event,
+                )
 
-    def _handle_workflow_partial_success_event(
+                yield iter_start_resp
-        self,
-        event: QueueWorkflowPartialSuccessEvent,
-        *,
-        graph_runtime_state: Optional[GraphRuntimeState] = None,
-        trace_manager: Optional[TraceQueueManager] = None,
-        **kwargs,
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle workflow partial success events."""
-        self._ensure_workflow_initialized()
-        validated_state = self._ensure_graph_runtime_initialized(graph_runtime_state)
-
-        with self._database_session() as session:
-            workflow_execution = self._workflow_cycle_manager.handle_workflow_run_partial_success(
-                workflow_run_id=self._workflow_run_id,
-                total_tokens=validated_state.total_tokens,
-                total_steps=validated_state.node_run_steps,
-                outputs=event.outputs,
-                exceptions_count=event.exceptions_count,
-                conversation_id=None,
-                trace_manager=trace_manager,
-                external_trace_id=self._application_generate_entity.extras.get("external_trace_id"),
-            )
 
-            # save workflow app log
+            elif isinstance(event, QueueIterationNextEvent):
-            self._save_workflow_app_log(session=session, workflow_execution=workflow_execution)
+                if not self._workflow_run_id:
+                    raise ValueError("workflow run not initialized.")
 
-            workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
+                iter_next_resp = self._workflow_response_converter.workflow_iteration_next_to_stream_response(
-                session=session,
+                    task_id=self._application_generate_entity.task_id,
-                task_id=self._application_generate_entity.task_id,
+                    workflow_execution_id=self._workflow_run_id,
-                workflow_execution=workflow_execution,
+                    event=event,
-            )
+                )
 
-        yield workflow_finish_resp
+                yield iter_next_resp
 
-    def _handle_workflow_failed_and_stop_events(
+            elif isinstance(event, QueueIterationCompletedEvent):
-        self,
+                if not self._workflow_run_id:
-        event: Union[QueueWorkflowFailedEvent, QueueStopEvent],
+                    raise ValueError("workflow run not initialized.")
-        *,
-        graph_runtime_state: Optional[GraphRuntimeState] = None,
-        trace_manager: Optional[TraceQueueManager] = None,
-        **kwargs,
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle workflow failed and stop events."""
-        self._ensure_workflow_initialized()
-        validated_state = self._ensure_graph_runtime_initialized(graph_runtime_state)
-
-        with self._database_session() as session:
-            workflow_execution = self._workflow_cycle_manager.handle_workflow_run_failed(
-                workflow_run_id=self._workflow_run_id,
-                total_tokens=validated_state.total_tokens,
-                total_steps=validated_state.node_run_steps,
-                status=WorkflowExecutionStatus.FAILED
-                if isinstance(event, QueueWorkflowFailedEvent)
-                else WorkflowExecutionStatus.STOPPED,
-                error_message=event.error if isinstance(event, QueueWorkflowFailedEvent) else event.get_stop_reason(),
-                conversation_id=None,
-                trace_manager=trace_manager,
-                exceptions_count=event.exceptions_count if isinstance(event, QueueWorkflowFailedEvent) else 0,
-                external_trace_id=self._application_generate_entity.extras.get("external_trace_id"),
-            )
 
-            # save workflow app log
+                iter_finish_resp = self._workflow_response_converter.workflow_iteration_completed_to_stream_response(
-            self._save_workflow_app_log(session=session, workflow_execution=workflow_execution)
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_execution_id=self._workflow_run_id,
+                    event=event,
+                )
 
-            workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
+                yield iter_finish_resp
-                session=session,
-                task_id=self._application_generate_entity.task_id,
-                workflow_execution=workflow_execution,
-            )
 
-        yield workflow_finish_resp
+            elif isinstance(event, QueueLoopStartEvent):
+                if not self._workflow_run_id:
+                    raise ValueError("workflow run not initialized.")
 
-    def _handle_text_chunk_event(
+                loop_start_resp = self._workflow_response_converter.workflow_loop_start_to_stream_response(
-        self,
+                    task_id=self._application_generate_entity.task_id,
-        event: QueueTextChunkEvent,
+                    workflow_execution_id=self._workflow_run_id,
-        *,
+                    event=event,
-        tts_publisher: Optional[AppGeneratorTTSPublisher] = None,
+                )
-        queue_message: Optional[Union[WorkflowQueueMessage, MessageQueueMessage]] = None,
-        **kwargs,
-    ) -> Generator[StreamResponse, None, None]:
-        """Handle text chunk events."""
-        delta_text = event.text
-        if delta_text is None:
-            return
 
-        # only publish tts message at text chunk streaming
+                yield loop_start_resp
-        if tts_publisher and queue_message:
-            tts_publisher.publish(queue_message)
 
-        yield self._text_chunk_to_stream_response(delta_text, from_variable_selector=event.from_variable_selector)
+            elif isinstance(event, QueueLoopNextEvent):
+                if not self._workflow_run_id:
+                    raise ValueError("workflow run not initialized.")
 
-    def _handle_agent_log_event(self, event: QueueAgentLogEvent, **kwargs) -> Generator[StreamResponse, None, None]:
+                loop_next_resp = self._workflow_response_converter.workflow_loop_next_to_stream_response(
-        """Handle agent log events."""
+                    task_id=self._application_generate_entity.task_id,
-        yield self._workflow_response_converter.handle_agent_log(
+                    workflow_execution_id=self._workflow_run_id,
-            task_id=self._application_generate_entity.task_id, event=event
+                    event=event,
-        )
+                )
 
-    def _get_event_handlers(self) -> dict[type, Callable]:
+                yield loop_next_resp
-        """Get mapping of event types to their handlers using fluent pattern."""
-        return {
-            # Basic events
-            QueuePingEvent: self._handle_ping_event,
-            QueueErrorEvent: self._handle_error_event,
-            QueueTextChunkEvent: self._handle_text_chunk_event,
-            # Workflow events
-            QueueWorkflowStartedEvent: self._handle_workflow_started_event,
-            QueueWorkflowSucceededEvent: self._handle_workflow_succeeded_event,
-            QueueWorkflowPartialSuccessEvent: self._handle_workflow_partial_success_event,
-            # Node events
-            QueueNodeRetryEvent: self._handle_node_retry_event,
-            QueueNodeStartedEvent: self._handle_node_started_event,
-            QueueNodeSucceededEvent: self._handle_node_succeeded_event,
-            # Parallel branch events
-            QueueParallelBranchRunStartedEvent: self._handle_parallel_branch_started_event,
-            # Iteration events
-            QueueIterationStartEvent: self._handle_iteration_start_event,
-            QueueIterationNextEvent: self._handle_iteration_next_event,
-            QueueIterationCompletedEvent: self._handle_iteration_completed_event,
-            # Loop events
-            QueueLoopStartEvent: self._handle_loop_start_event,
-            QueueLoopNextEvent: self._handle_loop_next_event,
-            QueueLoopCompletedEvent: self._handle_loop_completed_event,
-            # Agent events
-            QueueAgentLogEvent: self._handle_agent_log_event,
-        }
-
-    def _dispatch_event(
-        self,
-        event: Any,
-        *,
-        graph_runtime_state: Optional[GraphRuntimeState] = None,
-        tts_publisher: Optional[AppGeneratorTTSPublisher] = None,
-        trace_manager: Optional[TraceQueueManager] = None,
-        queue_message: Optional[Union[WorkflowQueueMessage, MessageQueueMessage]] = None,
-    ) -> Generator[StreamResponse, None, None]:
-        """Dispatch events using elegant pattern matching."""
-        handlers = self._get_event_handlers()
-        event_type = type(event)
 
-        # Direct handler lookup
+            elif isinstance(event, QueueLoopCompletedEvent):
-        if handler := handlers.get(event_type):
+                if not self._workflow_run_id:
-            yield from handler(
+                    raise ValueError("workflow run not initialized.")
-                event,
-                graph_runtime_state=graph_runtime_state,
-                tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
-                queue_message=queue_message,
-            )
-            return
 
-        # Handle node failure events with isinstance check
+                loop_finish_resp = self._workflow_response_converter.workflow_loop_completed_to_stream_response(
-        if isinstance(
+                    task_id=self._application_generate_entity.task_id,
-            event,
+                    workflow_execution_id=self._workflow_run_id,
-            (
+                    event=event,
-                QueueNodeFailedEvent,
+                )
-                QueueNodeInIterationFailedEvent,
-                QueueNodeInLoopFailedEvent,
-                QueueNodeExceptionEvent,
-            ),
-        ):
-            yield from self._handle_node_failed_events(
-                event,
-                graph_runtime_state=graph_runtime_state,
-                tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
-                queue_message=queue_message,
-            )
-            return
 
-        # Handle parallel branch finished events with isinstance check
+                yield loop_finish_resp
-        if isinstance(event, (QueueParallelBranchRunSucceededEvent, QueueParallelBranchRunFailedEvent)):
+
-            yield from self._handle_parallel_branch_finished_events(
+            elif isinstance(event, QueueWorkflowSucceededEvent):
-                event,
+                if not self._workflow_run_id:
-                graph_runtime_state=graph_runtime_state,
+                    raise ValueError("workflow run not initialized.")
-                tts_publisher=tts_publisher,
+                if not graph_runtime_state:
-                trace_manager=trace_manager,
+                    raise ValueError("graph runtime state not initialized.")
-                queue_message=queue_message,
+
-            )
+                with Session(db.engine, expire_on_commit=False) as session:
-            return
+                    workflow_execution = self._workflow_cycle_manager.handle_workflow_run_success(
+                        workflow_run_id=self._workflow_run_id,
+                        total_tokens=graph_runtime_state.total_tokens,
+                        total_steps=graph_runtime_state.node_run_steps,
+                        outputs=event.outputs,
+                        conversation_id=None,
+                        trace_manager=trace_manager,
+                    )
 
-        # Handle workflow failed and stop events with isinstance check
+                    # save workflow app log
-        if isinstance(event, (QueueWorkflowFailedEvent, QueueStopEvent)):
+                    self._save_workflow_app_log(session=session, workflow_execution=workflow_execution)
-            yield from self._handle_workflow_failed_and_stop_events(
-                event,
-                graph_runtime_state=graph_runtime_state,
-                tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
-                queue_message=queue_message,
-            )
-            return
 
-        # For unhandled events, we continue (original behavior)
+                    workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
-        return
+                        session=session,
+                        task_id=self._application_generate_entity.task_id,
+                        workflow_execution=workflow_execution,
+                    )
+                    session.commit()
+
+                yield workflow_finish_resp
+            elif isinstance(event, QueueWorkflowPartialSuccessEvent):
+                if not self._workflow_run_id:
+                    raise ValueError("workflow run not initialized.")
+                if not graph_runtime_state:
+                    raise ValueError("graph runtime state not initialized.")
+
+                with Session(db.engine, expire_on_commit=False) as session:
+                    workflow_execution = self._workflow_cycle_manager.handle_workflow_run_partial_success(
+                        workflow_run_id=self._workflow_run_id,
+                        total_tokens=graph_runtime_state.total_tokens,
+                        total_steps=graph_runtime_state.node_run_steps,
+                        outputs=event.outputs,
+                        exceptions_count=event.exceptions_count,
+                        conversation_id=None,
+                        trace_manager=trace_manager,
+                    )
 
-    def _process_stream_response(
+                    # save workflow app log
-        self,
+                    self._save_workflow_app_log(session=session, workflow_execution=workflow_execution)
-        tts_publisher: Optional[AppGeneratorTTSPublisher] = None,
-        trace_manager: Optional[TraceQueueManager] = None,
-    ) -> Generator[StreamResponse, None, None]:
-        """
-        Process stream response using elegant Fluent Python patterns.
-        Maintains exact same functionality as original 44-if-statement version.
-        """
-        # Initialize graph runtime state
-        graph_runtime_state = None
 
-        for queue_message in self._base_task_pipeline._queue_manager.listen():
+                    workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
-            event = queue_message.event
+                        session=session,
+                        task_id=self._application_generate_entity.task_id,
+                        workflow_execution=workflow_execution,
+                    )
+                    session.commit()
+
+                yield workflow_finish_resp
+            elif isinstance(event, QueueWorkflowFailedEvent | QueueStopEvent):
+                if not self._workflow_run_id:
+                    raise ValueError("workflow run not initialized.")
+                if not graph_runtime_state:
+                    raise ValueError("graph runtime state not initialized.")
+
+                with Session(db.engine, expire_on_commit=False) as session:
+                    workflow_execution = self._workflow_cycle_manager.handle_workflow_run_failed(
+                        workflow_run_id=self._workflow_run_id,
+                        total_tokens=graph_runtime_state.total_tokens,
+                        total_steps=graph_runtime_state.node_run_steps,
+                        status=WorkflowExecutionStatus.FAILED
+                        if isinstance(event, QueueWorkflowFailedEvent)
+                        else WorkflowExecutionStatus.STOPPED,
+                        error_message=event.error
+                        if isinstance(event, QueueWorkflowFailedEvent)
+                        else event.get_stop_reason(),
+                        conversation_id=None,
+                        trace_manager=trace_manager,
+                        exceptions_count=event.exceptions_count if isinstance(event, QueueWorkflowFailedEvent) else 0,
+                    )
 
-            match event:
+                    # save workflow app log
-                case QueueWorkflowStartedEvent():
+                    self._save_workflow_app_log(session=session, workflow_execution=workflow_execution)
-                    graph_runtime_state = event.graph_runtime_state
-                    yield from self._handle_workflow_started_event(event)
 
-                case QueueTextChunkEvent():
+                    workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
-                    yield from self._handle_text_chunk_event(
+                        session=session,
-                        event, tts_publisher=tts_publisher, queue_message=queue_message
+                        task_id=self._application_generate_entity.task_id,
+                        workflow_execution=workflow_execution,
                     )
+                    session.commit()
 
-                case QueueErrorEvent():
+                yield workflow_finish_resp
-                    yield from self._handle_error_event(event)
+            elif isinstance(event, QueueTextChunkEvent):
-                    break
+                delta_text = event.text
+                if delta_text is None:
+                    continue
 
-                # Handle all other events through elegant dispatch
+                # only publish tts message at text chunk streaming
-                case _:
+                if tts_publisher:
-                    if responses := list(
+                    tts_publisher.publish(queue_message)
-                        self._dispatch_event(
+
-                            event,
+                yield self._text_chunk_to_stream_response(
-                            graph_runtime_state=graph_runtime_state,
+                    delta_text, from_variable_selector=event.from_variable_selector
-                            tts_publisher=tts_publisher,
+                )
-                            trace_manager=trace_manager,
+            elif isinstance(event, QueueAgentLogEvent):
-                            queue_message=queue_message,
+                yield self._workflow_response_converter.handle_agent_log(
-                        )
+                    task_id=self._application_generate_entity.task_id, event=event
-                    ):
+                )
-                        yield from responses
+            else:
+                continue
 
         if tts_publisher:
             tts_publisher.publish(None)
 
     def _save_workflow_app_log(self, *, session: Session, workflow_execution: WorkflowExecution) -> None:
+        workflow_run = session.scalar(select(WorkflowRun).where(WorkflowRun.id == workflow_execution.id_))
+        assert workflow_run is not None
         invoke_from = self._application_generate_entity.invoke_from
         if invoke_from == InvokeFrom.SERVICE_API:
             created_from = WorkflowAppLogCreatedFrom.SERVICE_API
@@ -757,10 +576,10 @@ class WorkflowAppGenerateTaskPipeline:
             return
 
         workflow_app_log = WorkflowAppLog()
-        workflow_app_log.tenant_id = self._application_generate_entity.app_config.tenant_id
+        workflow_app_log.tenant_id = workflow_run.tenant_id
-        workflow_app_log.app_id = self._application_generate_entity.app_config.app_id
+        workflow_app_log.app_id = workflow_run.app_id
-        workflow_app_log.workflow_id = workflow_execution.workflow_id
+        workflow_app_log.workflow_id = workflow_run.workflow_id
-        workflow_app_log.workflow_run_id = workflow_execution.id_
+        workflow_app_log.workflow_run_id = workflow_run.id
         workflow_app_log.created_from = created_from.value
         workflow_app_log.created_by_role = self._created_by_role
         workflow_app_log.created_by = self._user_id