Merge branch 'main' of github.com:HaiyangPeng/dify into main

a more concise and effective extractor for excel and csv files.

.devcontainer/post_create_command.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-npm add -g pnpm@10.8.0
+npm add -g pnpm@10.11.1
 cd web && pnpm install
 pipx install uv
 

api/core/app/app_config/easy_ui_based_app/model_config/converter.py

@@ -70,7 +70,7 @@ class ModelConfigConverter:
         if not model_mode:
             model_mode = LLMMode.CHAT.value
             if model_schema and model_schema.model_properties.get(ModelPropertyKey.MODE):
-                model_mode = LLMMode.value_of(model_schema.model_properties[ModelPropertyKey.MODE]).value
+                model_mode = LLMMode(model_schema.model_properties[ModelPropertyKey.MODE]).value
 
         if not model_schema:
             raise ValueError(f"Model {model_name} not exist.")

api/core/helper/code_executor/code_executor.py

@@ -15,6 +15,7 @@ from core.helper.code_executor.python3.python3_transformer import Python3Templat
 from core.helper.code_executor.template_transformer import TemplateTransformer
 
 logger = logging.getLogger(__name__)
+code_execution_endpoint_url = URL(str(dify_config.CODE_EXECUTION_ENDPOINT))
 
 
 class CodeExecutionError(Exception):
@@ -64,7 +65,7 @@ class CodeExecutor:
         :param code: code
         :return:
         """
-        url = URL(str(dify_config.CODE_EXECUTION_ENDPOINT)) / "v1" / "sandbox" / "run"
+        url = code_execution_endpoint_url / "v1" / "sandbox" / "run"
 
         headers = {"X-Api-Key": dify_config.CODE_EXECUTION_API_KEY}
 

api/core/helper/marketplace.py

@@ -7,29 +7,28 @@ from configs import dify_config
 from core.helper.download import download_with_size_limit
 from core.plugin.entities.marketplace import MarketplacePluginDeclaration
 
+marketplace_api_url = URL(str(dify_config.MARKETPLACE_API_URL))
 
-def get_plugin_pkg_url(plugin_unique_identifier: str):
+
-    return (URL(str(dify_config.MARKETPLACE_API_URL)) / "api/v1/plugins/download").with_query(
+def get_plugin_pkg_url(plugin_unique_identifier: str) -> str:
-        unique_identifier=plugin_unique_identifier
+    return str((marketplace_api_url / "api/v1/plugins/download").with_query(unique_identifier=plugin_unique_identifier))
-    )
 
 
 def download_plugin_pkg(plugin_unique_identifier: str):
-    url = str(get_plugin_pkg_url(plugin_unique_identifier))
+    return download_with_size_limit(get_plugin_pkg_url(plugin_unique_identifier), dify_config.PLUGIN_MAX_PACKAGE_SIZE)
-    return download_with_size_limit(url, dify_config.PLUGIN_MAX_PACKAGE_SIZE)
 
 
 def batch_fetch_plugin_manifests(plugin_ids: list[str]) -> Sequence[MarketplacePluginDeclaration]:
     if len(plugin_ids) == 0:
         return []
 
-    url = str(URL(str(dify_config.MARKETPLACE_API_URL)) / "api/v1/plugins/batch")
+    url = str(marketplace_api_url / "api/v1/plugins/batch")
     response = requests.post(url, json={"plugin_ids": plugin_ids})
     response.raise_for_status()
     return [MarketplacePluginDeclaration(**plugin) for plugin in response.json()["data"]["plugins"]]
 
 
 def record_install_plugin_event(plugin_unique_identifier: str):
-    url = str(URL(str(dify_config.MARKETPLACE_API_URL)) / "api/v1/stats/plugins/install_count")
+    url = str(marketplace_api_url / "api/v1/stats/plugins/install_count")
     response = requests.post(url, json={"unique_identifier": plugin_unique_identifier})
     response.raise_for_status()

api/core/model_runtime/entities/llm_entities.py

@@ -17,19 +17,6 @@ class LLMMode(StrEnum):
     COMPLETION = "completion"
     CHAT = "chat"
 
-    @classmethod
-    def value_of(cls, value: str) -> "LLMMode":
-        """
-        Get value of given mode.
-
-        :param value: mode value
-        :return: mode
-        """
-        for mode in cls:
-            if mode.value == value:
-                return mode
-        raise ValueError(f"invalid mode value {value}")
-
 
 class LLMUsage(ModelUsage):
     """

api/core/plugin/impl/base.py

@@ -31,8 +31,7 @@ from core.plugin.impl.exc import (
     PluginUniqueIdentifierError,
 )
 
-plugin_daemon_inner_api_baseurl = dify_config.PLUGIN_DAEMON_URL
+plugin_daemon_inner_api_baseurl = URL(str(dify_config.PLUGIN_DAEMON_URL))
-plugin_daemon_inner_api_key = dify_config.PLUGIN_DAEMON_KEY
 
 T = TypeVar("T", bound=(BaseModel | dict | list | bool | str))
 
@@ -53,9 +52,9 @@ class BasePluginClient:
         """
         Make a request to the plugin daemon inner API.
         """
-        url = URL(str(plugin_daemon_inner_api_baseurl)) / path
+        url = plugin_daemon_inner_api_baseurl / path
         headers = headers or {}
-        headers["X-Api-Key"] = plugin_daemon_inner_api_key
+        headers["X-Api-Key"] = dify_config.PLUGIN_DAEMON_KEY
         headers["Accept-Encoding"] = "gzip, deflate, br"
 
         if headers.get("Content-Type") == "application/json" and isinstance(data, dict):

api/core/rag/datasource/vdb/milvus/milvus_vector.py

@@ -97,6 +97,10 @@ class MilvusVector(BaseVector):
 
         try:
             milvus_version = self._client.get_server_version()
+            # Check if it's Zilliz Cloud - it supports full-text search with Milvus 2.5 compatibility
+            if "Zilliz Cloud" in milvus_version:
+                return True
+            # For standard Milvus installations, check version number
             return version.parse(milvus_version).base_version >= version.parse("2.5.0").base_version
         except Exception as e:
             logger.warning(f"Failed to check Milvus version: {str(e)}. Disabling hybrid search.")

api/core/tools/custom_tool/tool.py

@@ -168,7 +168,7 @@ class ApiTool(Tool):
                 cookies[parameter["name"]] = value
 
             elif parameter["in"] == "header":
-                headers[parameter["name"]] = value
+                headers[parameter["name"]] = str(value)
 
         # check if there is a request body and handle it
         if "requestBody" in self.api_bundle.openapi and self.api_bundle.openapi["requestBody"] is not None:

api/core/tools/utils/parser.py

@@ -55,6 +55,13 @@ class ApiBasedToolSchemaParser:
             # convert parameters
             parameters = []
             if "parameters" in interface["operation"]:
+                for i, parameter in enumerate(interface["operation"]["parameters"]):
+                    if "$ref" in parameter:
+                        root = openapi
+                        reference = parameter["$ref"].split("/")[1:]
+                        for ref in reference:
+                            root = root[ref]
+                        interface["operation"]["parameters"][i] = root
                 for parameter in interface["operation"]["parameters"]:
                     tool_parameter = ToolParameter(
                         name=parameter["name"],

api/core/workflow/graph_engine/graph_engine.py

@@ -53,7 +53,6 @@ from core.workflow.nodes.end.end_stream_processor import EndStreamProcessor
 from core.workflow.nodes.enums import ErrorStrategy, FailBranchSourceHandle
 from core.workflow.nodes.event import RunCompletedEvent, RunRetrieverResourceEvent, RunStreamChunkEvent
 from core.workflow.nodes.node_mapping import NODE_TYPE_CLASSES_MAPPING
-from extensions.ext_database import db
 from models.enums import UserFrom
 from models.workflow import WorkflowType
 
@@ -607,8 +606,6 @@ class GraphEngine:
                         error=str(e),
                     )
                 )
-            finally:
-                db.session.remove()
 
     def _run_node(
         self,
@@ -646,7 +643,6 @@ class GraphEngine:
             agent_strategy=agent_strategy,
         )
 
-        db.session.close()
         max_retries = node_instance.node_data.retry_config.max_retries
         retry_interval = node_instance.node_data.retry_config.retry_interval_seconds
         retries = 0
@@ -863,8 +859,6 @@ class GraphEngine:
             except Exception as e:
                 logger.exception(f"Node {node_instance.node_data.title} run failed")
                 raise e
-            finally:
-                db.session.close()
 
     def _append_variables_recursively(self, node_id: str, variable_key_list: list[str], variable_value: VariableValue):
         """

api/core/workflow/nodes/agent/agent_node.py

@@ -2,6 +2,9 @@ import json
 from collections.abc import Generator, Mapping, Sequence
 from typing import Any, Optional, cast
 
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
 from core.agent.entities import AgentToolEntity
 from core.agent.plugin_entities import AgentStrategyParameter
 from core.memory.token_buffer_memory import TokenBufferMemory
@@ -320,12 +323,9 @@ class AgentNode(ToolNode):
             return None
         conversation_id = conversation_id_variable.value
 
-        # get conversation
+        with Session(db.engine, expire_on_commit=False) as session:
-        conversation = (
+            stmt = select(Conversation).where(Conversation.app_id == self.app_id, Conversation.id == conversation_id)
-            db.session.query(Conversation)
+            conversation = session.scalar(stmt)
-            .filter(Conversation.app_id == self.app_id, Conversation.id == conversation_id)
-            .first()
-        )
 
             if not conversation:
                 return None

api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -8,6 +8,7 @@ from typing import Any, Optional, cast
 
 from sqlalchemy import Float, and_, func, or_, text
 from sqlalchemy import cast as sqlalchemy_cast
+from sqlalchemy.orm import Session
 
 from core.app.app_config.entities import DatasetRetrieveConfigEntity
 from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
@@ -85,8 +86,8 @@ class KnowledgeRetrievalNode(LLMNode):
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error="Query is required."
             )
+        # TODO(-LAN-): Move this check outside.
         # check rate limit
-        if self.tenant_id:
         knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(self.tenant_id)
         if knowledge_rate_limit.enabled:
             current_time = int(time.time() * 1000)
@@ -95,14 +96,15 @@ class KnowledgeRetrievalNode(LLMNode):
             redis_client.zremrangebyscore(key, 0, current_time - 60000)
             request_count = redis_client.zcard(key)
             if request_count > knowledge_rate_limit.limit:
+                with Session(db.engine) as session:
                     # add ratelimit record
                     rate_limit_log = RateLimitLog(
                         tenant_id=self.tenant_id,
                         subscription_plan=knowledge_rate_limit.subscription_plan,
                         operation="knowledge",
                     )
-                    db.session.add(rate_limit_log)
+                    session.add(rate_limit_log)
-                    db.session.commit()
+                    session.commit()
                 return NodeRunResult(
                     status=WorkflowNodeExecutionStatus.FAILED,
                     inputs=variables,
@@ -173,7 +175,9 @@ class KnowledgeRetrievalNode(LLMNode):
         dataset_retrieval = DatasetRetrieval()
         if node_data.retrieval_mode == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE.value:
             # fetch model config
-            model_instance, model_config = self._fetch_model_config(node_data.single_retrieval_config.model)  # type: ignore
+            if node_data.single_retrieval_config is None:
+                raise ValueError("single_retrieval_config is required")
+            model_instance, model_config = self.get_model_config(node_data.single_retrieval_config.model)
             # check model is support tool calling
             model_type_instance = model_config.provider_model_bundle.model_type_instance
             model_type_instance = cast(LargeLanguageModel, model_type_instance)
@@ -424,7 +428,7 @@ class KnowledgeRetrievalNode(LLMNode):
             raise ValueError("metadata_model_config is required")
         # get metadata model instance
         # fetch model config
-        model_instance, model_config = self._fetch_model_config(node_data.metadata_model_config)  # type: ignore
+        model_instance, model_config = self.get_model_config(metadata_model_config)
         # fetch prompt messages
         prompt_template = self._get_prompt_template(
             node_data=node_data,
@@ -550,14 +554,7 @@ class KnowledgeRetrievalNode(LLMNode):
         variable_mapping[node_id + ".query"] = node_data.query_variable_selector
         return variable_mapping
 
-    def _fetch_model_config(self, model: ModelConfig) -> tuple[ModelInstance, ModelConfigWithCredentialsEntity]:  # type: ignore
+    def get_model_config(self, model: ModelConfig) -> tuple[ModelInstance, ModelConfigWithCredentialsEntity]:
-        """
-        Fetch model config
-        :param model: model
-        :return:
-        """
-        if model is None:
-            raise ValueError("model is required")
         model_name = model.name
         provider_name = model.provider
 

api/core/workflow/nodes/llm/node.py

@@ -7,6 +7,8 @@ from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any, Optional, cast
 
 import json_repair
+from sqlalchemy import select, update
+from sqlalchemy.orm import Session
 
 from configs import dify_config
 from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
@@ -303,8 +305,6 @@ class LLMNode(BaseNode[LLMNodeData]):
         prompt_messages: Sequence[PromptMessage],
         stop: Optional[Sequence[str]] = None,
     ) -> Generator[NodeEvent, None, None]:
-        db.session.close()
-
         invoke_result = model_instance.invoke_llm(
             prompt_messages=list(prompt_messages),
             model_parameters=node_data_model.completion_params,
@@ -603,13 +603,9 @@ class LLMNode(BaseNode[LLMNodeData]):
             return None
         conversation_id = conversation_id_variable.value
 
-        # get conversation
+        with Session(db.engine, expire_on_commit=False) as session:
-        conversation = (
+            stmt = select(Conversation).where(Conversation.app_id == self.app_id, Conversation.id == conversation_id)
-            db.session.query(Conversation)
+            conversation = session.scalar(stmt)
-            .filter(Conversation.app_id == self.app_id, Conversation.id == conversation_id)
-            .first()
-        )
-
             if not conversation:
                 return None
 
@@ -847,20 +843,24 @@ class LLMNode(BaseNode[LLMNodeData]):
                 used_quota = 1
 
         if used_quota is not None and system_configuration.current_quota_type is not None:
-            db.session.query(Provider).filter(
+            with Session(db.engine) as session:
+                stmt = (
+                    update(Provider)
+                    .where(
                         Provider.tenant_id == tenant_id,
                         # TODO: Use provider name with prefix after the data migration.
                         Provider.provider_name == ModelProviderID(model_instance.provider).provider_name,
                         Provider.provider_type == ProviderType.SYSTEM.value,
                         Provider.quota_type == system_configuration.current_quota_type.value,
                         Provider.quota_limit > Provider.quota_used,
-            ).update(
-                {
-                    "quota_used": Provider.quota_used + used_quota,
-                    "last_used": datetime.now(tz=UTC).replace(tzinfo=None),
-                }
                     )
-            db.session.commit()
+                    .values(
+                        quota_used=Provider.quota_used + used_quota,
+                        last_used=datetime.now(tz=UTC).replace(tzinfo=None),
+                    )
+                )
+                session.execute(stmt)
+                session.commit()
 
     @classmethod
     def _extract_variable_selector_to_variable_mapping(

api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py

@@ -31,7 +31,6 @@ from core.workflow.entities.workflow_node_execution import WorkflowNodeExecution
 from core.workflow.nodes.enums import NodeType
 from core.workflow.nodes.llm import LLMNode, ModelConfig
 from core.workflow.utils import variable_template_parser
-from extensions.ext_database import db
 
 from .entities import ParameterExtractorNodeData
 from .exc import (
@@ -259,8 +258,6 @@ class ParameterExtractorNode(LLMNode):
         tools: list[PromptMessageTool],
         stop: list[str],
     ) -> tuple[str, LLMUsage, Optional[AssistantPromptMessage.ToolCall]]:
-        db.session.close()
-
         invoke_result = model_instance.invoke_llm(
             prompt_messages=prompt_messages,
             model_parameters=node_data_model.completion_params,

api/core/workflow/nodes/question_classifier/question_classifier_node.py

@@ -79,9 +79,13 @@ class QuestionClassifierNode(LLMNode):
             memory=memory,
             max_token_limit=rest_token,
         )
+        # Some models (e.g. Gemma, Mistral) force roles alternation (user/assistant/user/assistant...).
+        # If both self._get_prompt_template and self._fetch_prompt_messages append a user prompt,
+        # two consecutive user prompts will be generated, causing model's error.
+        # To avoid this, set sys_query to an empty string so that only one user prompt is appended at the end.
         prompt_messages, stop = self._fetch_prompt_messages(
             prompt_template=prompt_template,
-            sys_query=query,
+            sys_query="",
             memory=memory,
             model_config=model_config,
             sys_files=files,

api/core/workflow/nodes/variable_aggregator/entities.py

@@ -1,7 +1,8 @@
-from typing import Literal, Optional
+from typing import Optional
 
 from pydantic import BaseModel
 
+from core.variables.types import SegmentType
 from core.workflow.nodes.base import BaseNodeData
 
 
@@ -17,7 +18,7 @@ class AdvancedSettings(BaseModel):
         Group.
         """
 
-        output_type: Literal["string", "number", "object", "array[string]", "array[number]", "array[object]"]
+        output_type: SegmentType
         variables: list[list[str]]
         group_name: str
 

api/libs/smtp.py

@@ -28,7 +28,8 @@ class SMTPClient:
             else:
                 smtp = smtplib.SMTP(self.server, self.port, timeout=10)
 
-            if self.username and self.password:
+            # Only authenticate if both username and password are non-empty
+            if self.username and self.password and self.username.strip() and self.password.strip():
                 smtp.login(self.username, self.password)
 
             msg = MIMEMultipart()

api/pyproject.toml

@@ -14,7 +14,7 @@ dependencies = [
     "chardet~=5.1.0",
     "flask~=3.1.0",
     "flask-compress~=1.17",
-    "flask-cors~=5.0.0",
+    "flask-cors~=6.0.0",
     "flask-login~=0.6.3",
     "flask-migrate~=4.0.7",
     "flask-restful~=0.3.10",
@@ -36,7 +36,6 @@ dependencies = [
     "mailchimp-transactional~=1.0.50",
     "markdown~=3.5.1",
     "numpy~=1.26.4",
-    "oci~=2.135.1",
     "openai~=1.61.0",
     "openpyxl~=3.1.5",
     "opik~=1.7.25",
@@ -143,13 +142,16 @@ dev = [
     "types-requests~=2.32.0",
     "types-requests-oauthlib~=2.0.0",
     "types-shapely~=2.0.0",
-    "types-simplejson~=3.20.0",
+    "types-simplejson>=3.20.0",
-    "types-six~=1.17.0",
+    "types-six>=1.17.0",
-    "types-tensorflow~=2.18.0",
+    "types-tensorflow>=2.18.0",
-    "types-tqdm~=4.67.0",
+    "types-tqdm>=4.67.0",
-    "types-ujson~=5.10.0",
+    "types-ujson>=5.10.0",
     "boto3-stubs>=1.38.20",
     "types-jmespath>=1.0.2.20240106",
+    "types_pyOpenSSL>=24.1.0",
+    "types_cffi>=1.17.0",
+    "types_setuptools>=80.9.0",
 ]
 
 ############################################################

web/Dockerfile

@@ -6,7 +6,7 @@ LABEL maintainer="takatost@gmail.com"
 # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
 
 RUN apk add --no-cache tzdata
-RUN npm install -g pnpm@10.8.0
+RUN npm install -g pnpm@10.11.1
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 

web/app/components/base/chat/chat/hooks.ts

@@ -366,8 +366,9 @@ export const useChat = (
             if (!newResponseItem)
               return
 
+            const isUseAgentThought = newResponseItem.agent_thoughts?.length > 0
             updateChatTreeNode(responseItem.id, {
-              content: newResponseItem.answer,
+              content: isUseAgentThought ? '' : newResponseItem.answer,
               log: [
                 ...newResponseItem.message,
                 ...(newResponseItem.message[newResponseItem.message.length - 1].role !== 'assistant'

web/app/components/base/markdown/markdown-utils.ts

@@ -33,5 +33,6 @@ export const preprocessThinkTag = (content: string) => {
   return flow([
     (str: string) => str.replace(thinkOpenTagRegex, '<details data-think=true>\n'),
     (str: string) => str.replace(thinkCloseTagRegex, '\n[ENDTHINKFLAG]</details>'),
+    (str: string) => str.replace(/(<\/details>)(?![^\S\r\n]*[\r\n])(?![^\S\r\n]*$)/g, '$1\n'),
   ])(content)
 }

web/app/components/base/radio/component/radio/index.tsx

@@ -37,14 +37,15 @@ export default function Radio({
   const isChecked = groupContext ? groupContext.value === value : checked
   const divClassName = `
     flex items-center py-1 relative
-    px-7 cursor-pointer hover:bg-gray-200 rounded
+    px-7 cursor-pointer text-text-secondary rounded
+    bg-components-option-card-option-bg hover:bg-components-option-card-option-bg-hover hover:shadow-xs
   `
 
   return (
     <div className={cn(
       s.label,
       disabled ? s.disabled : '',
-      isChecked ? 'bg-white shadow' : '',
+      isChecked ? 'bg-components-option-card-option-bg-hover shadow-xs' : '',
       divClassName,
       className)}
     onClick={() => handleChange(value)}

web/app/components/workflow/nodes/_base/components/editor/code-editor/editor-support-vars.tsx

@@ -148,7 +148,7 @@ const CodeEditor: FC<Props> = ({
       {isShowVarPicker && (
         <div
           ref={popupRef}
-          className='w-[228px] space-y-1 rounded-lg border border-gray-200 bg-white p-1 shadow-lg'
+          className='w-[228px] space-y-1 rounded-lg border border-components-panel-border bg-components-panel-bg p-1 shadow-lg'
           style={{
             position: 'fixed',
             top: popupPosition.y,

web/app/components/workflow/nodes/_base/components/variable/var-type-picker.tsx

@@ -43,7 +43,7 @@ const VarReferencePicker: FC<Props> = ({
         offset={4}
       >
         <PortalToFollowElemTrigger onClick={() => setOpen(!open)} className='w-[120px] cursor-pointer'>
-          <div className='flex h-8 items-center justify-between rounded-lg border-0 bg-components-button-secondary-bg px-2.5 text-[13px] text-text-primary'>
+          <div className='flex h-8 items-center justify-between rounded-lg border-0 bg-components-input-bg-normal px-2.5 text-[13px] text-text-primary'>
             <div className='w-0 grow truncate capitalize' title={value}>{value}</div>
             <RiArrowDownSLine className='h-3.5 w-3.5 shrink-0 text-text-secondary' />
           </div>

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-number.tsx

@@ -3,7 +3,7 @@ import { useTranslation } from 'react-i18next'
 import ConditionValueMethod from './condition-value-method'
 import type { ConditionValueMethodProps } from './condition-value-method'
 import ConditionVariableSelector from './condition-variable-selector'
-import ConditionCommonVariableSelector from './condition-common-variable-selector.tsx'
+import ConditionCommonVariableSelector from './condition-common-variable-selector'
 import type {
   Node,
   NodeOutPutVar,

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-string.tsx

@@ -3,7 +3,7 @@ import { useTranslation } from 'react-i18next'
 import ConditionValueMethod from './condition-value-method'
 import type { ConditionValueMethodProps } from './condition-value-method'
 import ConditionVariableSelector from './condition-variable-selector'
-import ConditionCommonVariableSelector from './condition-common-variable-selector.tsx'
+import ConditionCommonVariableSelector from './condition-common-variable-selector'
 import type {
   Node,
   NodeOutPutVar,