From f76f70f0b6119a448f6d7ddf0c27d00009ab060f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B9=9B=E9=9C=B2=E5=85=88=E7=94=9F?= Date: Thu, 5 Jun 2025 23:05:50 +0800 Subject: [PATCH 01/12] Fix builtin_providers for tools. (#20697) Signed-off-by: zhanluxianshen --- api/core/tools/builtin_tool/_position.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/core/tools/builtin_tool/_position.yaml b/api/core/tools/builtin_tool/_position.yaml index b5875e2075..0e811de311 100644 --- a/api/core/tools/builtin_tool/_position.yaml +++ b/api/core/tools/builtin_tool/_position.yaml @@ -1,3 +1,4 @@ +- audio - code - time -- qrcode +- webscraper From 138ad6e8b33368a66a1956ca5ddc087d5928317e Mon Sep 17 00:00:00 2001 From: jefferyvvv <33647240+jefferyvvv@users.noreply.github.com> Date: Thu, 5 Jun 2025 23:09:00 +0800 Subject: [PATCH 02/12] fix: opensearch fulltext search with metadata filtering dsl error (#20702) Co-authored-by: wenjun.gu --- api/core/rag/datasource/vdb/opensearch/opensearch_vector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py index 6991598ce6..b156be1a22 100644 --- a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py +++ b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py @@ -209,10 +209,10 @@ class OpenSearchVector(BaseVector): return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: - full_text_query = {"query": {"match": {Field.CONTENT_KEY.value: query}}} + full_text_query = {"query": {"bool": {"must": [{"match": {Field.CONTENT_KEY.value: query}}]}}} document_ids_filter = kwargs.get("document_ids_filter") if document_ids_filter: - full_text_query["query"]["terms"] = {"metadata.document_id": document_ids_filter} + full_text_query["query"]["bool"]["filter"] = [{"terms": {"metadata.document_id": document_ids_filter}}] response = self._client.search(index=self._collection_name.lower(), body=full_text_query) From 38554c5f3e50eb9a5f28c06d1ecdca669ce8e3c8 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Thu, 5 Jun 2025 23:36:33 +0800 Subject: [PATCH 03/12] fix(inner_api/plugin/wraps): refresh user model after creation in get user function (#20704) --- api/controllers/inner_api/plugin/wraps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/inner_api/plugin/wraps.py b/api/controllers/inner_api/plugin/wraps.py index b2849a7962..50408e0929 100644 --- a/api/controllers/inner_api/plugin/wraps.py +++ b/api/controllers/inner_api/plugin/wraps.py @@ -32,6 +32,7 @@ def get_user(tenant_id: str, user_id: str | None) -> Account | EndUser: ) session.add(user_model) session.commit() + session.refresh(user_model) else: user_model = AccountService.load_user(user_id) if not user_model: From 4f14d7c0ca49e44e7b4d0fc1623b2beff9a440ec Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 6 Jun 2025 09:09:31 +0800 Subject: [PATCH 04/12] chore: bump uv to 0.7.x (#20692) --- .github/actions/setup-uv/action.yml | 2 +- api/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-uv/action.yml b/.github/actions/setup-uv/action.yml index a596be63f7..0499b44dba 100644 --- a/.github/actions/setup-uv/action.yml +++ b/.github/actions/setup-uv/action.yml @@ -8,7 +8,7 @@ inputs: uv-version: description: UV version to set up required: true - default: '0.6.14' + default: '~=0.7.11' uv-lockfile: description: Path to the UV lockfile to restore cache from required: true diff --git a/api/Dockerfile b/api/Dockerfile index cff696ff56..7e4997507f 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -4,7 +4,7 @@ FROM python:3.12-slim-bookworm AS base WORKDIR /app/api # Install uv -ENV UV_VERSION=0.6.14 +ENV UV_VERSION=0.7.11 RUN pip install --no-cache-dir uv==${UV_VERSION} From 4271602cfca984aa999f505cf2e30d43c61ad280 Mon Sep 17 00:00:00 2001 From: jefferyvvv <33647240+jefferyvvv@users.noreply.github.com> Date: Fri, 6 Jun 2025 09:10:01 +0800 Subject: [PATCH 05/12] fix: opensearch metadata filtering returns empty (#20701) Co-authored-by: wenjun.gu Co-authored-by: crazywoola <427733928@qq.com> --- api/core/rag/datasource/vdb/opensearch/opensearch_vector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py index b156be1a22..6a6c2b73ef 100644 --- a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py +++ b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py @@ -255,7 +255,8 @@ class OpenSearchVector(BaseVector): Field.METADATA_KEY.value: { "type": "object", "properties": { - "doc_id": {"type": "keyword"} # Map doc_id to keyword type + "doc_id": {"type": "keyword"}, # Map doc_id to keyword type + "document_id": {"type": "keyword"}, }, }, } From 4f0c9fdf2b02ac4ad0b3fa4a85abf92f3531025e Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 6 Jun 2025 10:44:21 +0800 Subject: [PATCH 06/12] chore: remove repeat public api and service api panel (#20715) --- .../[appId]/overview/chartView.tsx | 32 +++++++++++-------- .../[appId]/overview/page.tsx | 8 ++--- .../[appId]/overview/tracing/panel.tsx | 4 +-- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx index 4afba06eae..32822e3315 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx @@ -18,9 +18,10 @@ const queryDateFormat = 'YYYY-MM-DD HH:mm' export type IChartViewProps = { appId: string + headerRight: React.ReactNode } -export default function ChartView({ appId }: IChartViewProps) { +export default function ChartView({ appId, headerRight }: IChartViewProps) { const { t } = useTranslation() const appDetail = useAppStore(state => state.appDetail) const isChatApp = appDetail?.mode !== 'completion' && appDetail?.mode !== 'workflow' @@ -46,19 +47,22 @@ export default function ChartView({ appId }: IChartViewProps) { return (
-
- {t('appOverview.analysis.title')} - ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))} - className='mt-0 !w-40' - onSelect={(item) => { - const id = item.value - const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1' - const name = item.name || t('appLog.filter.period.allTime') - onSelect({ value, name }) - }} - defaultValue={'2'} - /> +
+
+ {t('appOverview.analysis.title')} + ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))} + className='mt-0 !w-40' + onSelect={(item) => { + const id = item.value + const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1' + const name = item.name || t('appLog.filter.period.allTime') + onSelect({ value, name }) + }} + defaultValue={'2'} + /> +
+ {headerRight}
{!isWorkflow && (
diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx index 0f1bb7e18d..fc97f5e669 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx @@ -1,6 +1,5 @@ import React from 'react' import ChartView from './chartView' -import CardView from './cardView' import TracingPanel from './tracing/panel' import ApikeyInfoPanel from '@/app/components/app/overview/apikey-info-panel' @@ -18,9 +17,10 @@ const Overview = async (props: IDevelopProps) => { return (
- - - + } + />
) } diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 8575117c41..bc85f3a734 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -154,7 +154,6 @@ const Panel: FC = () => { if (!isLoaded) { return (
- <div className='w-[200px]'> <Loading /> </div> @@ -163,8 +162,7 @@ const Panel: FC = () => { } return ( - <div className={cn('mb-3 flex items-center justify-between')}> - <Title className='h-[41px]' /> + <div className={cn('flex items-center justify-between')}> <div className={cn( 'flex cursor-pointer items-center rounded-xl border-l-[0.5px] border-t border-effects-highlight bg-background-default-dodge p-2 shadow-xs hover:border-effects-highlight-lightmode-off hover:bg-background-default-lighter', From c1a13fa553f3156f9b86adcdc4f7e8dd77025b88 Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Fri, 6 Jun 2025 10:48:28 +0800 Subject: [PATCH 07/12] chore: replace pseudo-random generators with secrets module (#20616) --- api/.ruff.toml | 1 + api/core/helper/moderation.py | 4 ++-- api/core/workflow/nodes/http_request/executor.py | 5 +++-- api/libs/helper.py | 4 ++-- api/services/account_service.py | 7 +++---- api/services/dataset_service.py | 4 ++-- api/services/webapp_auth_service.py | 4 ++-- api/tests/unit_tests/core/helper/test_ssrf_proxy.py | 4 ++-- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/api/.ruff.toml b/api/.ruff.toml index 41a24abad9..facb0d5419 100644 --- a/api/.ruff.toml +++ b/api/.ruff.toml @@ -43,6 +43,7 @@ select = [ "S307", # suspicious-eval-usage, disallow use of `eval` and `ast.literal_eval` "S301", # suspicious-pickle-usage, disallow use of `pickle` and its wrappers. "S302", # suspicious-marshal-usage, disallow use of `marshal` module + "S311", # suspicious-non-cryptographic-random-usage ] ignore = [ diff --git a/api/core/helper/moderation.py b/api/core/helper/moderation.py index 6a5982eca4..a324ac2767 100644 --- a/api/core/helper/moderation.py +++ b/api/core/helper/moderation.py @@ -1,5 +1,5 @@ import logging -import random +import secrets from typing import cast from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity @@ -38,7 +38,7 @@ def check_moderation(tenant_id: str, model_config: ModelConfigWithCredentialsEnt if len(text_chunks) == 0: return True - text_chunk = random.choice(text_chunks) + text_chunk = secrets.choice(text_chunks) try: model_provider_factory = ModelProviderFactory(tenant_id) diff --git a/api/core/workflow/nodes/http_request/executor.py b/api/core/workflow/nodes/http_request/executor.py index e28ac6343b..2c83b00d4a 100644 --- a/api/core/workflow/nodes/http_request/executor.py +++ b/api/core/workflow/nodes/http_request/executor.py @@ -1,8 +1,9 @@ import base64 import json +import secrets +import string from collections.abc import Mapping from copy import deepcopy -from random import randint from typing import Any, Literal from urllib.parse import urlencode, urlparse @@ -434,4 +435,4 @@ def _generate_random_string(n: int) -> str: >>> _generate_random_string(5) 'abcde' """ - return "".join([chr(randint(97, 122)) for _ in range(n)]) + return "".join(secrets.choice(string.ascii_lowercase) for _ in range(n)) diff --git a/api/libs/helper.py b/api/libs/helper.py index 463ba3308b..e78a782fbe 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -1,7 +1,7 @@ import json import logging -import random import re +import secrets import string import subprocess import time @@ -176,7 +176,7 @@ def generate_string(n): letters_digits = string.ascii_letters + string.digits result = "" for i in range(n): - result += random.choice(letters_digits) + result += secrets.choice(letters_digits) return result diff --git a/api/services/account_service.py b/api/services/account_service.py index ac84a46299..14d238467d 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -1,7 +1,6 @@ import base64 import json import logging -import random import secrets import uuid from datetime import UTC, datetime, timedelta @@ -261,7 +260,7 @@ class AccountService: @staticmethod def generate_account_deletion_verification_code(account: Account) -> tuple[str, str]: - code = "".join([str(random.randint(0, 9)) for _ in range(6)]) + code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)]) token = TokenManager.generate_token( account=account, token_type="account_deletion", additional_data={"code": code} ) @@ -429,7 +428,7 @@ class AccountService: additional_data: dict[str, Any] = {}, ): if not code: - code = "".join([str(random.randint(0, 9)) for _ in range(6)]) + code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)]) additional_data["code"] = code token = TokenManager.generate_token( account=account, email=email, token_type="reset_password", additional_data=additional_data @@ -456,7 +455,7 @@ class AccountService: raise EmailCodeLoginRateLimitExceededError() - code = "".join([str(random.randint(0, 9)) for _ in range(6)]) + code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)]) token = TokenManager.generate_token( account=account, email=email, token_type="email_code_login", additional_data={"code": code} ) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 4a5e9b3520..e98b47921f 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -2,7 +2,7 @@ import copy import datetime import json import logging -import random +import secrets import time import uuid from collections import Counter @@ -970,7 +970,7 @@ class DocumentService: documents.append(document) batch = document.batch else: - batch = time.strftime("%Y%m%d%H%M%S") + str(random.randint(100000, 999999)) + batch = time.strftime("%Y%m%d%H%M%S") + str(100000 + secrets.randbelow(exclusive_upper_bound=900000)) # save process rule if not dataset_process_rule: process_rule = knowledge_config.process_rule diff --git a/api/services/webapp_auth_service.py b/api/services/webapp_auth_service.py index 79d5217de7..d83303056b 100644 --- a/api/services/webapp_auth_service.py +++ b/api/services/webapp_auth_service.py @@ -1,4 +1,4 @@ -import random +import secrets from datetime import UTC, datetime, timedelta from typing import Any, Optional, cast @@ -66,7 +66,7 @@ class WebAppAuthService: if email is None: raise ValueError("Email must be provided.") - code = "".join([str(random.randint(0, 9)) for _ in range(6)]) + code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)]) token = TokenManager.generate_token( account=account, email=email, token_type="webapp_email_code_login", additional_data={"code": code} ) diff --git a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py index c688d3952b..37749f0c66 100644 --- a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py +++ b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py @@ -1,4 +1,4 @@ -import random +import secrets from unittest.mock import MagicMock, patch import pytest @@ -34,7 +34,7 @@ def test_retry_logic_success(mock_request): side_effects = [] for _ in range(SSRF_DEFAULT_MAX_RETRIES): - status_code = random.choice(STATUS_FORCELIST) + status_code = secrets.choice(STATUS_FORCELIST) mock_response = MagicMock() mock_response.status_code = status_code side_effects.append(mock_response) From 85859b67237c28dbec67f73b7c8e7e8484c62800 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Fri, 6 Jun 2025 10:53:57 +0800 Subject: [PATCH 08/12] feat: add browser list (#20717) --- web/package.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/web/package.json b/web/package.json index affbef9382..ff4214f966 100644 --- a/web/package.json +++ b/web/package.json @@ -5,6 +5,18 @@ "engines": { "node": ">=v22.11.0" }, + "browserslist": [ + "last 1 Chrome version", + "last 1 Firefox version", + "last 1 Edge version", + "last 1 Safari version", + "iOS >=15", + "Android >= 10", + "and_chr >= 126", + "and_ff >= 137", + "and_uc >= 15.5", + "and_qq >= 14.9" + ], "scripts": { "dev": "cross-env NODE_OPTIONS='--inspect' next dev", "build": "next build", From 723b69cf8d64bd69045934d74163b0b68f2f3859 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Fri, 6 Jun 2025 16:15:37 +0800 Subject: [PATCH 09/12] chore: chart panel ui enhance (#20743) --- .../[appId]/overview/chartView.tsx | 32 ++++++++++--------- .../[appId]/overview/tracing/panel.tsx | 13 -------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx index 32822e3315..646c8bd93d 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx @@ -47,22 +47,24 @@ export default function ChartView({ appId, headerRight }: IChartViewProps) { return ( <div> - <div className='mb-4 flex items-center justify-between'> - <div className='system-xl-semibold flex flex-row items-center text-text-primary'> - <span className='mr-3'>{t('appOverview.analysis.title')}</span> - <SimpleSelect - items={Object.entries(TIME_PERIOD_MAPPING).map(([k, v]) => ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))} - className='mt-0 !w-40' - onSelect={(item) => { - const id = item.value - const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1' - const name = item.name || t('appLog.filter.period.allTime') - onSelect({ value, name }) - }} - defaultValue={'2'} - /> + <div className='mb-4'> + <div className='system-xl-semibold mb-2 text-text-primary'>{t('common.appMenus.overview')}</div> + <div className='flex items-center justify-between'> + <div className='flex flex-row items-center'> + <SimpleSelect + items={Object.entries(TIME_PERIOD_MAPPING).map(([k, v]) => ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))} + className='mt-0 !w-40' + onSelect={(item) => { + const id = item.value + const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1' + const name = item.name || t('appLog.filter.period.allTime') + onSelect({ value, name }) + }} + defaultValue={'2'} + /> + </div> + {headerRight} </div> - {headerRight} </div> {!isWorkflow && ( <div className='mb-6 grid w-full grid-cols-1 gap-6 xl:grid-cols-2'> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index bc85f3a734..76e90ecf19 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -23,19 +23,6 @@ import Divider from '@/app/components/base/divider' const I18N_PREFIX = 'app.tracing' -const Title = ({ - className, -}: { - className?: string -}) => { - const { t } = useTranslation() - - return ( - <div className={cn('system-xl-semibold flex items-center text-text-primary', className)}> - {t('common.appMenus.overview')} - </div> - ) -} const Panel: FC = () => { const { t } = useTranslation() const pathname = usePathname() From 37c3283450c9bec0d8b44d2335be85603d0f294c Mon Sep 17 00:00:00 2001 From: jefferyvvv <33647240+jefferyvvv@users.noreply.github.com> Date: Fri, 6 Jun 2025 16:29:15 +0800 Subject: [PATCH 10/12] fix: opensearch vector search falls back to keyword search (#20723) Co-authored-by: wenjun.gu <wenjun.gu@envision-energy.com> --- .../datasource/vdb/opensearch/opensearch_vector.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py index 6a6c2b73ef..0abb3c0077 100644 --- a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py +++ b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py @@ -184,7 +184,16 @@ class OpenSearchVector(BaseVector): } document_ids_filter = kwargs.get("document_ids_filter") if document_ids_filter: - query["query"] = {"terms": {"metadata.document_id": document_ids_filter}} + query["query"] = { + "script_score": { + "query": {"bool": {"filter": [{"terms": {Field.DOCUMENT_ID.value: document_ids_filter}}]}}, + "script": { + "source": "knn_score", + "lang": "knn", + "params": {"field": Field.VECTOR.value, "query_value": query_vector, "space_type": "l2"}, + }, + } + } try: response = self._client.search(index=self._collection_name.lower(), body=query) From 0c8447fd0e48851f862f050e951296ad39844071 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Fri, 6 Jun 2025 16:44:36 +0800 Subject: [PATCH 11/12] fix: missing bot name in orchestrate (#20747) --- web/app/components/base/chat/chat/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/base/chat/chat/index.tsx b/web/app/components/base/chat/chat/index.tsx index c0842af0c4..801daa6589 100644 --- a/web/app/components/base/chat/chat/index.tsx +++ b/web/app/components/base/chat/chat/index.tsx @@ -303,7 +303,7 @@ const Chat: FC<ChatProps> = ({ { !noChatInput && ( <ChatInputArea - botName={appData?.site.title || ''} + botName={appData?.site.title || 'Bot'} disabled={inputDisabled} showFeatureBar={showFeatureBar} showFileUpload={showFileUpload} From 930c4cb60904f29b21f0cf5abf8da8063ed4ec7a Mon Sep 17 00:00:00 2001 From: QuantumGhost <obelisk.reg+git@gmail.com> Date: Fri, 6 Jun 2025 21:03:59 +0800 Subject: [PATCH 12/12] feat(api): Adjust `WorkflowDraftVariable` and `WorkflowNodeExecutionModel` (#20746) - Add `node_execution_id` column to `WorkflowDraftVariable`, allowing efficient implementation of the "Reset to last run value" feature. - Add additional index for `WorkflowNodeExecutionModel` to improve the performance of last run lookup. Closes #20745. --- ...w_draft_varaibles_add_node_execution_id.py | 60 +++++++++++++ api/models/workflow.py | 87 +++++++++++++------ 2 files changed, 121 insertions(+), 26 deletions(-) create mode 100644 api/migrations/versions/2025_06_06_1424-4474872b0ee6_workflow_draft_varaibles_add_node_execution_id.py diff --git a/api/migrations/versions/2025_06_06_1424-4474872b0ee6_workflow_draft_varaibles_add_node_execution_id.py b/api/migrations/versions/2025_06_06_1424-4474872b0ee6_workflow_draft_varaibles_add_node_execution_id.py new file mode 100644 index 0000000000..d7a5d116c9 --- /dev/null +++ b/api/migrations/versions/2025_06_06_1424-4474872b0ee6_workflow_draft_varaibles_add_node_execution_id.py @@ -0,0 +1,60 @@ +"""`workflow_draft_varaibles` add `node_execution_id` column, add an index for `workflow_node_executions`. + +Revision ID: 4474872b0ee6 +Revises: 2adcbe1f5dfb +Create Date: 2025-06-06 14:24:44.213018 + +""" +from alembic import op +import models as models +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4474872b0ee6' +down_revision = '2adcbe1f5dfb' +branch_labels = None +depends_on = None + + +def upgrade(): + # `CREATE INDEX CONCURRENTLY` cannot run within a transaction, so use the `autocommit_block` + # context manager to wrap the index creation statement. + # Reference: + # + # - https://www.postgresql.org/docs/current/sql-createindex.html#:~:text=Another%20difference%20is,CREATE%20INDEX%20CONCURRENTLY%20cannot. + # - https://alembic.sqlalchemy.org/en/latest/api/runtime.html#alembic.runtime.migration.MigrationContext.autocommit_block + with op.get_context().autocommit_block(): + op.create_index( + op.f('workflow_node_executions_tenant_id_idx'), + "workflow_node_executions", + ['tenant_id', 'workflow_id', 'node_id', sa.literal_column('created_at DESC')], + unique=False, + postgresql_concurrently=True, + ) + + with op.batch_alter_table('workflow_draft_variables', schema=None) as batch_op: + batch_op.add_column(sa.Column('node_execution_id', models.types.StringUUID(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + # `DROP INDEX CONCURRENTLY` cannot run within a transaction, so use the `autocommit_block` + # context manager to wrap the index creation statement. + # Reference: + # + # - https://www.postgresql.org/docs/current/sql-createindex.html#:~:text=Another%20difference%20is,CREATE%20INDEX%20CONCURRENTLY%20cannot. + # - https://alembic.sqlalchemy.org/en/latest/api/runtime.html#alembic.runtime.migration.MigrationContext.autocommit_block + # `DROP INDEX CONCURRENTLY` cannot run within a transaction, so commit existing transactions first. + # Reference: + # + # https://www.postgresql.org/docs/current/sql-createindex.html#:~:text=Another%20difference%20is,CREATE%20INDEX%20CONCURRENTLY%20cannot. + with op.get_context().autocommit_block(): + op.drop_index(op.f('workflow_node_executions_tenant_id_idx'), postgresql_concurrently=True) + + with op.batch_alter_table('workflow_draft_variables', schema=None) as batch_op: + batch_op.drop_column('node_execution_id') + + # ### end Alembic commands ### diff --git a/api/models/workflow.py b/api/models/workflow.py index e868fb77a7..2fff045543 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -16,8 +16,8 @@ if TYPE_CHECKING: from models.model import AppMode import sqlalchemy as sa -from sqlalchemy import UniqueConstraint, func -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy import Index, PrimaryKeyConstraint, UniqueConstraint, func +from sqlalchemy.orm import Mapped, declared_attr, mapped_column from constants import DEFAULT_FILE_NUMBER_LIMITS, HIDDEN_VALUE from core.helper import encrypter @@ -590,28 +590,48 @@ class WorkflowNodeExecutionModel(Base): """ __tablename__ = "workflow_node_executions" - __table_args__ = ( - db.PrimaryKeyConstraint("id", name="workflow_node_execution_pkey"), - db.Index( - "workflow_node_execution_workflow_run_idx", - "tenant_id", - "app_id", - "workflow_id", - "triggered_from", - "workflow_run_id", - ), - db.Index( - "workflow_node_execution_node_run_idx", "tenant_id", "app_id", "workflow_id", "triggered_from", "node_id" - ), - db.Index( - "workflow_node_execution_id_idx", - "tenant_id", - "app_id", - "workflow_id", - "triggered_from", - "node_execution_id", - ), - ) + + @declared_attr + def __table_args__(cls): # noqa + return ( + PrimaryKeyConstraint("id", name="workflow_node_execution_pkey"), + Index( + "workflow_node_execution_workflow_run_idx", + "tenant_id", + "app_id", + "workflow_id", + "triggered_from", + "workflow_run_id", + ), + Index( + "workflow_node_execution_node_run_idx", + "tenant_id", + "app_id", + "workflow_id", + "triggered_from", + "node_id", + ), + Index( + "workflow_node_execution_id_idx", + "tenant_id", + "app_id", + "workflow_id", + "triggered_from", + "node_execution_id", + ), + Index( + # The first argument is the index name, + # which we leave as `None`` to allow auto-generation by the ORM. + None, + cls.tenant_id, + cls.workflow_id, + cls.node_id, + # MyPy may flag the following line because it doesn't recognize that + # the `declared_attr` decorator passes the receiving class as the first + # argument to this method, allowing us to reference class attributes. + cls.created_at.desc(), # type: ignore + ), + ) id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()")) tenant_id: Mapped[str] = mapped_column(StringUUID) @@ -885,14 +905,29 @@ class WorkflowDraftVariable(Base): selector: Mapped[str] = mapped_column(sa.String(255), nullable=False, name="selector") + # The data type of this variable's value value_type: Mapped[SegmentType] = mapped_column(EnumText(SegmentType, length=20)) - # JSON string + + # The variable's value serialized as a JSON string value: Mapped[str] = mapped_column(sa.Text, nullable=False, name="value") - # visible + # Controls whether the variable should be displayed in the variable inspection panel visible: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=True) + + # Determines whether this variable can be modified by users editable: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False) + # The `node_execution_id` field identifies the workflow node execution that created this variable. + # It corresponds to the `id` field in the `WorkflowNodeExecutionModel` model. + # + # This field is not `None` for system variables and node variables, and is `None` + # for conversation variables. + node_execution_id: Mapped[str | None] = mapped_column( + StringUUID, + nullable=True, + default=None, + ) + def get_selector(self) -> list[str]: selector = json.loads(self.selector) if not isinstance(selector, list):