Merge pull request #2 from kenwoodjw/main

merge main

api/commands.py

@@ -818,8 +818,9 @@ def clear_free_plan_tenant_expired_logs(days: int, batch: int, tenant_ids: list[
     click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green"))
 
 
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
 @click.command("clear-orphaned-file-records", help="Clear orphaned file records.")
-def clear_orphaned_file_records():
+def clear_orphaned_file_records(force: bool):
     """
     Clear orphaned file records in the database.
     """
@@ -845,7 +846,15 @@ def clear_orphaned_file_records():
 
     # notify user and ask for confirmation
     click.echo(
-        click.style("This command will find and delete orphaned file records in the following tables:", fg="yellow")
+        click.style(
+            "This command will first find and delete orphaned file records from the message_files table,", fg="yellow"
+        )
+    )
+    click.echo(
+        click.style(
+            "and then it will find and delete orphaned file records in the following tables:",
+            fg="yellow",
+        )
     )
     for files_table in files_tables:
         click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
@@ -878,11 +887,55 @@ def clear_orphaned_file_records():
             fg="yellow",
         )
     )
-    click.confirm("Do you want to proceed?", abort=True)
+    if not force:
+        click.confirm("Do you want to proceed?", abort=True)
 
     # start the cleanup process
     click.echo(click.style("Starting orphaned file records cleanup.", fg="white"))
 
+    # clean up the orphaned records in the message_files table where message_id doesn't exist in messages table
+    try:
+        click.echo(
+            click.style("- Listing message_files records where message_id doesn't exist in messages table", fg="white")
+        )
+        query = (
+            "SELECT mf.id, mf.message_id "
+            "FROM message_files mf LEFT JOIN messages m ON mf.message_id = m.id "
+            "WHERE m.id IS NULL"
+        )
+        orphaned_message_files = []
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(query))
+            for i in rs:
+                orphaned_message_files.append({"id": str(i[0]), "message_id": str(i[1])})
+
+        if orphaned_message_files:
+            click.echo(click.style(f"Found {len(orphaned_message_files)} orphaned message_files records:", fg="white"))
+            for record in orphaned_message_files:
+                click.echo(click.style(f"  - id: {record['id']}, message_id: {record['message_id']}", fg="black"))
+
+            if not force:
+                click.confirm(
+                    (
+                        f"Do you want to proceed "
+                        f"to delete all {len(orphaned_message_files)} orphaned message_files records?"
+                    ),
+                    abort=True,
+                )
+
+            click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
+            query = "DELETE FROM message_files WHERE id IN :ids"
+            with db.engine.begin() as conn:
+                conn.execute(db.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+            click.echo(
+                click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
+            )
+        else:
+            click.echo(click.style("No orphaned message_files records found. There is nothing to delete.", fg="green"))
+    except Exception as e:
+        click.echo(click.style(f"Error deleting orphaned message_files records: {str(e)}", fg="red"))
+
+    # clean up the orphaned records in the rest of the *_files tables
     try:
         # fetch file id and keys from each table
         all_files_in_tables = []
@@ -964,7 +1017,8 @@ def clear_orphaned_file_records():
     click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white"))
     for file in orphaned_files:
         click.echo(click.style(f"- orphaned file id: {file}", fg="black"))
-    click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
+    if not force:
+        click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
 
     # delete orphaned records for each file
     try:
@@ -979,8 +1033,9 @@ def clear_orphaned_file_records():
     click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green"))
 
 
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
 @click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.")
-def remove_orphaned_files_on_storage():
+def remove_orphaned_files_on_storage(force: bool):
     """
     Remove orphaned files on the storage.
     """
@@ -1028,7 +1083,8 @@ def remove_orphaned_files_on_storage():
             fg="yellow",
         )
     )
-    click.confirm("Do you want to proceed?", abort=True)
+    if not force:
+        click.confirm("Do you want to proceed?", abort=True)
 
     # start the cleanup process
     click.echo(click.style("Starting orphaned files cleanup.", fg="white"))
@@ -1069,7 +1125,8 @@ def remove_orphaned_files_on_storage():
     click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white"))
     for file in orphaned_files:
         click.echo(click.style(f"- orphaned file: {file}", fg="black"))
-    click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
+    if not force:
+        click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
 
     # delete orphaned files
     removed_files = 0

api/configs/middleware/vdb/opensearch_config.py

@@ -1,4 +1,5 @@
-from typing import Optional
+import enum
+from typing import Literal, Optional
 
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
@@ -9,6 +10,14 @@ class OpenSearchConfig(BaseSettings):
     Configuration settings for OpenSearch
     """
 
+    class AuthMethod(enum.StrEnum):
+        """
+        Authentication method for OpenSearch
+        """
+
+        BASIC = "basic"
+        AWS_MANAGED_IAM = "aws_managed_iam"
+
     OPENSEARCH_HOST: Optional[str] = Field(
         description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
@@ -19,6 +28,16 @@ class OpenSearchConfig(BaseSettings):
         default=9200,
     )
 
+    OPENSEARCH_SECURE: bool = Field(
+        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
+        default=False,
+    )
+
+    OPENSEARCH_AUTH_METHOD: AuthMethod = Field(
+        description="Authentication method for OpenSearch connection (default is 'basic')",
+        default=AuthMethod.BASIC,
+    )
+
     OPENSEARCH_USER: Optional[str] = Field(
         description="Username for authenticating with OpenSearch",
         default=None,
@@ -29,7 +48,11 @@ class OpenSearchConfig(BaseSettings):
         default=None,
     )
 
-    OPENSEARCH_SECURE: bool = Field(
-        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
-        default=False,
+    OPENSEARCH_AWS_REGION: Optional[str] = Field(
+        description="AWS region for OpenSearch (e.g. 'us-west-2')",
+        default=None,
+    )
+
+    OPENSEARCH_AWS_SERVICE: Optional[Literal["es", "aoss"]] = Field(
+        description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
     )

api/controllers/files/image_preview.py

@@ -75,7 +75,7 @@ class FilePreviewApi(Resource):
         if args["as_attachment"]:
             encoded_filename = quote(upload_file.name)
             response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
-        response.headers["Content-Type"] = "application/octet-stream"
+            response.headers["Content-Type"] = "application/octet-stream"
 
         return response
 

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -381,6 +381,8 @@ class WorkflowCycleManage:
         workflow_node_execution.elapsed_time = elapsed_time
         workflow_node_execution.execution_metadata = execution_metadata
 
+        self._workflow_node_execution_repository.update(workflow_node_execution)
+
         return workflow_node_execution
 
     def _handle_workflow_node_execution_retried(

api/core/llm_generator/llm_generator.py

@@ -3,6 +3,8 @@ import logging
 import re
 from typing import Optional, cast
 
+import json_repair
+
 from core.llm_generator.output_parser.rule_config_generator import RuleConfigGeneratorOutputParser
 from core.llm_generator.output_parser.suggested_questions_after_answer import SuggestedQuestionsAfterAnswerOutputParser
 from core.llm_generator.prompts import (
@@ -366,7 +368,20 @@ class LLMGenerator:
                 ),
             )
 
-            generated_json_schema = cast(str, response.message.content)
+            raw_content = response.message.content
+
+            if not isinstance(raw_content, str):
+                raise ValueError(f"LLM response content must be a string, got: {type(raw_content)}")
+
+            try:
+                parsed_content = json.loads(raw_content)
+            except json.JSONDecodeError:
+                parsed_content = json_repair.loads(raw_content)
+
+            if not isinstance(parsed_content, dict | list):
+                raise ValueError(f"Failed to parse structured output from llm: {raw_content}")
+
+            generated_json_schema = json.dumps(parsed_content, indent=2, ensure_ascii=False)
             return {"output": generated_json_schema, "error": ""}
 
         except InvokeError as e:

api/core/rag/datasource/vdb/milvus/milvus_vector.py

@@ -27,8 +27,8 @@ class MilvusConfig(BaseModel):
 
     uri: str  # Milvus server URI
     token: Optional[str] = None  # Optional token for authentication
-    user: str  # Username for authentication
-    password: str  # Password for authentication
+    user: Optional[str] = None  # Username for authentication
+    password: Optional[str] = None  # Password for authentication
     batch_size: int = 100  # Batch size for operations
     database: str = "default"  # Database name
     enable_hybrid_search: bool = False  # Flag to enable hybrid search
@@ -43,10 +43,11 @@ class MilvusConfig(BaseModel):
         """
         if not values.get("uri"):
             raise ValueError("config MILVUS_URI is required")
-        if not values.get("user"):
-            raise ValueError("config MILVUS_USER is required")
-        if not values.get("password"):
-            raise ValueError("config MILVUS_PASSWORD is required")
+        if not values.get("token"):
+            if not values.get("user"):
+                raise ValueError("config MILVUS_USER is required")
+            if not values.get("password"):
+                raise ValueError("config MILVUS_PASSWORD is required")
         return values
 
     def to_milvus_params(self):
@@ -356,11 +357,14 @@ class MilvusVector(BaseVector):
                 )
             redis_client.set(collection_exist_cache_key, 1, ex=3600)
 
-    def _init_client(self, config) -> MilvusClient:
+    def _init_client(self, config: MilvusConfig) -> MilvusClient:
         """
         Initialize and return a Milvus client.
         """
-        client = MilvusClient(uri=config.uri, user=config.user, password=config.password, db_name=config.database)
+        if config.token:
+            client = MilvusClient(uri=config.uri, token=config.token, db_name=config.database)
+        else:
+            client = MilvusClient(uri=config.uri, user=config.user, password=config.password, db_name=config.database)
         return client
 
 

api/core/rag/datasource/vdb/opensearch/opensearch_vector.py

@@ -1,10 +1,9 @@
 import json
 import logging
-import ssl
-from typing import Any, Optional
+from typing import Any, Literal, Optional
 from uuid import uuid4
 
-from opensearchpy import OpenSearch, helpers
+from opensearchpy import OpenSearch, Urllib3AWSV4SignerAuth, Urllib3HttpConnection, helpers
 from opensearchpy.helpers import BulkIndexError
 from pydantic import BaseModel, model_validator
 
@@ -24,9 +23,12 @@ logger = logging.getLogger(__name__)
 class OpenSearchConfig(BaseModel):
     host: str
     port: int
+    secure: bool = False
+    auth_method: Literal["basic", "aws_managed_iam"] = "basic"
     user: Optional[str] = None
     password: Optional[str] = None
-    secure: bool = False
+    aws_region: Optional[str] = None
+    aws_service: Optional[str] = None
 
     @model_validator(mode="before")
     @classmethod
@@ -35,24 +37,40 @@ class OpenSearchConfig(BaseModel):
             raise ValueError("config OPENSEARCH_HOST is required")
         if not values.get("port"):
             raise ValueError("config OPENSEARCH_PORT is required")
+        if values.get("auth_method") == "aws_managed_iam":
+            if not values.get("aws_region"):
+                raise ValueError("config OPENSEARCH_AWS_REGION is required for AWS_MANAGED_IAM auth method")
+            if not values.get("aws_service"):
+                raise ValueError("config OPENSEARCH_AWS_SERVICE is required for AWS_MANAGED_IAM auth method")
         return values
 
-    def create_ssl_context(self) -> ssl.SSLContext:
-        ssl_context = ssl.create_default_context()
-        ssl_context.check_hostname = False
-        ssl_context.verify_mode = ssl.CERT_NONE  # Disable Certificate Validation
-        return ssl_context
+    def create_aws_managed_iam_auth(self) -> Urllib3AWSV4SignerAuth:
+        import boto3  # type: ignore
+
+        return Urllib3AWSV4SignerAuth(
+            credentials=boto3.Session().get_credentials(),
+            region=self.aws_region,
+            service=self.aws_service,  # type: ignore[arg-type]
+        )
 
     def to_opensearch_params(self) -> dict[str, Any]:
         params = {
             "hosts": [{"host": self.host, "port": self.port}],
             "use_ssl": self.secure,
             "verify_certs": self.secure,
+            "connection_class": Urllib3HttpConnection,
+            "pool_maxsize": 20,
         }
-        if self.user and self.password:
+
+        if self.auth_method == "basic":
+            logger.info("Using basic authentication for OpenSearch Vector DB")
+
             params["http_auth"] = (self.user, self.password)
-        if self.secure:
-            params["ssl_context"] = self.create_ssl_context()
+        elif self.auth_method == "aws_managed_iam":
+            logger.info("Using AWS managed IAM role for OpenSearch Vector DB")
+
+            params["http_auth"] = self.create_aws_managed_iam_auth()
+
         return params
 
 
@@ -76,16 +94,23 @@ class OpenSearchVector(BaseVector):
             action = {
                 "_op_type": "index",
                 "_index": self._collection_name.lower(),
-                "_id": uuid4().hex,
                 "_source": {
                     Field.CONTENT_KEY.value: documents[i].page_content,
                     Field.VECTOR.value: embeddings[i],  # Make sure you pass an array here
                     Field.METADATA_KEY.value: documents[i].metadata,
                 },
             }
+            # See https://github.com/langchain-ai/langchainjs/issues/4346#issuecomment-1935123377
+            if self._client_config.aws_service not in ["aoss"]:
+                action["_id"] = uuid4().hex
             actions.append(action)
 
-        helpers.bulk(self._client, actions)
+        helpers.bulk(
+            client=self._client,
+            actions=actions,
+            timeout=30,
+            max_retries=3,
+        )
 
     def get_ids_by_metadata_field(self, key: str, value: str):
         query = {"query": {"term": {f"{Field.METADATA_KEY.value}.{key}": value}}}
@@ -234,6 +259,7 @@ class OpenSearchVector(BaseVector):
                     },
                 }
 
+                logger.info(f"Creating OpenSearch index {self._collection_name.lower()}")
                 self._client.indices.create(index=self._collection_name.lower(), body=index_body)
 
             redis_client.set(collection_exist_cache_key, 1, ex=3600)
@@ -252,9 +278,12 @@ class OpenSearchVectorFactory(AbstractVectorFactory):
         open_search_config = OpenSearchConfig(
             host=dify_config.OPENSEARCH_HOST or "localhost",
             port=dify_config.OPENSEARCH_PORT,
+            secure=dify_config.OPENSEARCH_SECURE,
+            auth_method=dify_config.OPENSEARCH_AUTH_METHOD.value,
             user=dify_config.OPENSEARCH_USER,
             password=dify_config.OPENSEARCH_PASSWORD,
-            secure=dify_config.OPENSEARCH_SECURE,
+            aws_region=dify_config.OPENSEARCH_AWS_REGION,
+            aws_service=dify_config.OPENSEARCH_AWS_SERVICE,
         )
 
         return OpenSearchVector(collection_name=collection_name, config=open_search_config)

api/core/rag/rerank/rerank_model.py

@@ -52,14 +52,16 @@ class RerankModelRunner(BaseRerankRunner):
         rerank_documents = []
 
         for result in rerank_result.docs:
-            # format document
-            rerank_document = Document(
-                page_content=result.text,
-                metadata=documents[result.index].metadata,
-                provider=documents[result.index].provider,
-            )
-            if rerank_document.metadata is not None:
-                rerank_document.metadata["score"] = result.score
-                rerank_documents.append(rerank_document)
+            if score_threshold is None or result.score >= score_threshold:
+                # format document
+                rerank_document = Document(
+                    page_content=result.text,
+                    metadata=documents[result.index].metadata,
+                    provider=documents[result.index].provider,
+                )
+                if rerank_document.metadata is not None:
+                    rerank_document.metadata["score"] = result.score
+                    rerank_documents.append(rerank_document)
 
-        return rerank_documents
+        rerank_documents.sort(key=lambda x: x.metadata.get("score", 0.0), reverse=True)
+        return rerank_documents[:top_n] if top_n else rerank_documents

api/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dify-api"
-version = "1.3.1"
+dynamic = ["version"]
 requires-python = ">=3.11,<3.13"
 
 dependencies = [
@@ -89,8 +89,12 @@ dependencies = [
 # Before adding new dependency, consider place it in
 # alphabet order (a-z) and suitable group.
 
+[tool.setuptools]
+packages = []
+
 [tool.uv]
 default-groups = ["storage", "tools", "vdb"]
+package = false
 
 [dependency-groups]
 

api/tests/integration_tests/vdb/opensearch/test_opensearch.py

@@ -23,13 +23,70 @@ def setup_mock_redis():
     ext_redis.redis_client.lock = MagicMock(return_value=mock_redis_lock)
 
 
+class TestOpenSearchConfig:
+    def test_to_opensearch_params(self):
+        config = OpenSearchConfig(
+            host="localhost",
+            port=9200,
+            secure=True,
+            user="admin",
+            password="password",
+        )
+
+        params = config.to_opensearch_params()
+
+        assert params["hosts"] == [{"host": "localhost", "port": 9200}]
+        assert params["use_ssl"] is True
+        assert params["verify_certs"] is True
+        assert params["connection_class"].__name__ == "Urllib3HttpConnection"
+        assert params["http_auth"] == ("admin", "password")
+
+    @patch("boto3.Session")
+    @patch("core.rag.datasource.vdb.opensearch.opensearch_vector.Urllib3AWSV4SignerAuth")
+    def test_to_opensearch_params_with_aws_managed_iam(
+        self, mock_aws_signer_auth: MagicMock, mock_boto_session: MagicMock
+    ):
+        mock_credentials = MagicMock()
+        mock_boto_session.return_value.get_credentials.return_value = mock_credentials
+
+        mock_auth_instance = MagicMock()
+        mock_aws_signer_auth.return_value = mock_auth_instance
+
+        aws_region = "ap-southeast-2"
+        aws_service = "aoss"
+        host = f"aoss-endpoint.{aws_region}.aoss.amazonaws.com"
+        port = 9201
+
+        config = OpenSearchConfig(
+            host=host,
+            port=port,
+            secure=True,
+            auth_method="aws_managed_iam",
+            aws_region=aws_region,
+            aws_service=aws_service,
+        )
+
+        params = config.to_opensearch_params()
+
+        assert params["hosts"] == [{"host": host, "port": port}]
+        assert params["use_ssl"] is True
+        assert params["verify_certs"] is True
+        assert params["connection_class"].__name__ == "Urllib3HttpConnection"
+        assert params["http_auth"] is mock_auth_instance
+
+        mock_aws_signer_auth.assert_called_once_with(
+            credentials=mock_credentials, region=aws_region, service=aws_service
+        )
+        assert mock_boto_session.return_value.get_credentials.called
+
+
 class TestOpenSearchVector:
     def setup_method(self):
         self.collection_name = "test_collection"
         self.example_doc_id = "example_doc_id"
         self.vector = OpenSearchVector(
             collection_name=self.collection_name,
-            config=OpenSearchConfig(host="localhost", port=9200, user="admin", password="password", secure=False),
+            config=OpenSearchConfig(host="localhost", port=9200, secure=False, user="admin", password="password"),
         )
         self.vector._client = MagicMock()
 

api/tests/unit_tests/core/workflow/graph_engine/test_graph_engine.py

@@ -864,10 +864,11 @@ def test_condition_parallel_correct_output(mock_close, mock_remove, app):
             with patch.object(CodeNode, "_run", new=code_generator):
                 generator = graph_engine.run()
                 stream_content = ""
-                res_content = "VAT:\ndify 123"
+                wrong_content = ["Stamp Duty", "other"]
                 for item in generator:
                     if isinstance(item, NodeRunStreamChunkEvent):
                         stream_content += f"{item.chunk_content}\n"
                     if isinstance(item, GraphRunSucceededEvent):
-                        assert item.outputs == {"answer": res_content}
-                assert stream_content == res_content + "\n"
+                        assert item.outputs is not None
+                        answer = item.outputs["answer"]
+                        assert all(rc not in answer for rc in wrong_content)

api/uv.lock

@@ -1155,7 +1155,6 @@ wheels = [
 
 [[package]]
 name = "dify-api"
-version = "1.3.1"
 source = { virtual = "." }
 dependencies = [
     { name = "authlib" },

docker/.env.example

@@ -39,6 +39,12 @@ APP_WEB_URL=
 # File preview or download Url prefix.
 # used to display File preview or download Url to the front-end or as Multi-model inputs;
 # Url is signed and has expiration time.
+# Setting FILES_URL is required for file processing plugins.
+#   - For https://example.com, use FILES_URL=https://example.com
+#   - For http://example.com, use FILES_URL=http://example.com
+#   Recommendation: use a dedicated domain (e.g., https://upload.example.com).
+#   Alternatively, use http://<your-ip>:5001 or http://api:5001,
+#   ensuring port 5001 is externally accessible (see docker-compose.yaml).
 FILES_URL=
 
 # ------------------------------
@@ -520,9 +526,13 @@ RELYT_DATABASE=postgres
 # open search configuration, only available when VECTOR_STORE is `opensearch`
 OPENSEARCH_HOST=opensearch
 OPENSEARCH_PORT=9200
+OPENSEARCH_SECURE=true
+OPENSEARCH_AUTH_METHOD=basic
 OPENSEARCH_USER=admin
 OPENSEARCH_PASSWORD=admin
-OPENSEARCH_SECURE=true
+# If using AWS managed IAM, e.g. Managed Cluster or OpenSearch Serverless
+OPENSEARCH_AWS_REGION=ap-southeast-1
+OPENSEARCH_AWS_SERVICE=aoss
 
 # tencent vector configurations, only available when VECTOR_STORE is `tencent`
 TENCENT_VECTOR_DB_URL=http://127.0.0.1

docker/README.md

@@ -14,7 +14,6 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T
 
 - **Unified Vector Database Services**: All vector database services are now managed from a single Docker Compose file `docker-compose.yaml`. You can switch between different vector databases by setting the `VECTOR_STORE` environment variable in your `.env` file.
 - **Mandatory .env File**: A `.env` file is now required to run `docker compose up`. This file is crucial for configuring your deployment and for any custom settings to persist through upgrades.
-- **Legacy Support**: Previous deployment files are now located in the `docker-legacy` directory and will no longer be maintained.
 
 ### How to Deploy Dify with `docker-compose.yaml`
 

docker/docker-compose.yaml

@@ -225,9 +225,12 @@ x-shared-env: &shared-api-worker-env
   RELYT_DATABASE: ${RELYT_DATABASE:-postgres}
   OPENSEARCH_HOST: ${OPENSEARCH_HOST:-opensearch}
   OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
+  OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true}
+  OPENSEARCH_AUTH_METHOD: ${OPENSEARCH_AUTH_METHOD:-basic}
   OPENSEARCH_USER: ${OPENSEARCH_USER:-admin}
   OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD:-admin}
-  OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true}
+  OPENSEARCH_AWS_REGION: ${OPENSEARCH_AWS_REGION:-ap-southeast-1}
+  OPENSEARCH_AWS_SERVICE: ${OPENSEARCH_AWS_SERVICE:-aoss}
   TENCENT_VECTOR_DB_URL: ${TENCENT_VECTOR_DB_URL:-http://127.0.0.1}
   TENCENT_VECTOR_DB_API_KEY: ${TENCENT_VECTOR_DB_API_KEY:-dify}
   TENCENT_VECTOR_DB_TIMEOUT: ${TENCENT_VECTOR_DB_TIMEOUT:-30}

web/app/components/base/mermaid/utils.ts

@@ -22,10 +22,6 @@ export function preprocessMermaidCode(code: string): string {
     .replace(/section\s+([^:]+):/g, (match, sectionName) => `section ${sectionName}：`)
     // Fix common syntax issues
     .replace(/fifopacket/g, 'rect')
-    // Ensure graph has direction
-    .replace(/^graph\s+((?:TB|BT|RL|LR)*)/, (match, direction) => {
-      return direction ? match : 'graph TD'
-    })
     // Clean up empty lines and extra spaces
     .trim()
 }

web/app/components/datasets/create/website/no-data.tsx

@@ -17,6 +17,7 @@ type Props = {
 
 const NoData: FC<Props> = ({
   onConfig,
+  provider,
 }) => {
   const { t } = useTranslation()
 
@@ -38,7 +39,7 @@ const NoData: FC<Props> = ({
     } : null,
   }
 
-  const currentProvider = Object.values(providerConfig).find(provider => provider !== null) || providerConfig[DataSourceProvider.jinaReader]
+  const currentProvider = providerConfig[provider] || providerConfig[DataSourceProvider.jinaReader]
 
   if (!currentProvider) return null
 

web/app/components/header/nav/nav-selector/index.tsx

@@ -6,7 +6,7 @@ import {
   RiArrowDownSLine,
   RiArrowRightSLine,
 } from '@remixicon/react'
-import { Menu, MenuButton, MenuItems, Transition } from '@headlessui/react'
+import { Menu, MenuButton, MenuItem, MenuItems, Transition } from '@headlessui/react'
 import { useRouter } from 'next/navigation'
 import { debounce } from 'lodash-es'
 import cn from '@/utils/classnames'
@@ -77,7 +77,7 @@ const NavSelector = ({ curNav, navs, createText, isApp, onCreate, onLoadmore }:
               <div className="overflow-auto px-1 py-1" style={{ maxHeight: '50vh' }} onScroll={handleScroll}>
                 {
                   navs.map(nav => (
-                    <MenuItems key={nav.id}>
+                    <MenuItem key={nav.id}>
                       <div className='flex w-full cursor-pointer items-center truncate rounded-lg px-3 py-[6px] text-[14px] font-normal text-gray-700 hover:bg-gray-100' onClick={() => {
                         if (curNav?.id === nav.id)
                           return
@@ -112,12 +112,12 @@ const NavSelector = ({ curNav, navs, createText, isApp, onCreate, onLoadmore }:
                           {nav.name}
                         </div>
                       </div>
-                    </MenuItems>
+                    </MenuItem>
                   ))
                 }
               </div>
               {!isApp && isCurrentWorkspaceEditor && (
-                <MenuButton className='w-full p-1'>
+                <MenuItem as="div" className='w-full p-1'>
                   <div onClick={() => onCreate('')} className={cn(
                     'flex cursor-pointer items-center gap-2 rounded-lg px-3 py-[6px] hover:bg-gray-100',
                   )}>
@@ -126,7 +126,7 @@ const NavSelector = ({ curNav, navs, createText, isApp, onCreate, onLoadmore }:
                     </div>
                     <div className='grow text-left text-[14px] font-normal text-gray-700'>{createText}</div>
                   </div>
-                </MenuButton>
+                </MenuItem>
               )}
               {isApp && isCurrentWorkspaceEditor && (
                 <Menu as="div" className="relative h-full w-full">

web/docker/entrypoint.sh

@@ -32,4 +32,7 @@ export NEXT_PUBLIC_MAX_TOOLS_NUM=${MAX_TOOLS_NUM}
 export NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER=${ENABLE_WEBSITE_JINAREADER:-true}
 export NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL=${ENABLE_WEBSITE_FIRECRAWL:-true}
 export NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL=${ENABLE_WEBSITE_WATERCRAWL:-true}
+export NEXT_PUBLIC_LOOP_NODE_MAX_COUNT=${LOOP_NODE_MAX_COUNT}
+export NEXT_PUBLIC_MAX_PARALLEL_LIMIT=${MAX_PARALLEL_LIMIT}
+export NEXT_PUBLIC_MAX_ITERATIONS_NUM=${MAX_ITERATIONS_NUM}
 pm2 start /app/web/server.js --name dify-web --cwd /app/web -i ${PM2_INSTANCES} --no-daemon

web/public/embed.js

@@ -57,7 +57,7 @@
     left: unset;
     min-width: 24rem;
     width: 48%;
-    max-width: calc(100vw - 2rem);
+    max-width: 40rem; /* Match mobile breakpoint*/
     min-height: 43.75rem;
     height: 88%;
     max-height: calc(100vh - 6rem);

web/public/embed.min.js

@@ -39,4 +39,4 @@
     <svg id="closeIcon" style="display:none" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
       <path d="M18 18L6 6M6 18L18 6" stroke="white" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
     </svg>
-    `,n.appendChild(e),document.body.appendChild(n),n.addEventListener("click",t),n.addEventListener("touchend",t),y.draggable){var a=n;var l=y.dragAxis||"both";let s,d,t,r;function o(e){u=!1,r=("touchstart"===e.type?(s=e.touches[0].clientX-a.offsetLeft,d=e.touches[0].clientY-a.offsetTop,t=e.touches[0].clientX,e.touches[0]):(s=e.clientX-a.offsetLeft,d=e.clientY-a.offsetTop,t=e.clientX,e)).clientY,document.addEventListener("mousemove",i),document.addEventListener("touchmove",i,{passive:!1}),document.addEventListener("mouseup",c),document.addEventListener("touchend",c),e.preventDefault()}function i(n){var o="touchmove"===n.type?n.touches[0]:n,i=o.clientX-t,o=o.clientY-r;if(u=8<Math.abs(i)||8<Math.abs(o)?!0:u){a.style.transition="none",a.style.cursor="grabbing";i=document.getElementById(m);i&&(i.style.display="none",p("open"));let e,t;t="touchmove"===n.type?(e=n.touches[0].clientX-s,window.innerHeight-n.touches[0].clientY-d):(e=n.clientX-s,window.innerHeight-n.clientY-d);o=a.getBoundingClientRect(),i=window.innerWidth-o.width,n=window.innerHeight-o.height;"x"!==l&&"both"!==l||a.style.setProperty(`--${h}-left`,Math.max(0,Math.min(e,i))+"px"),"y"!==l&&"both"!==l||a.style.setProperty(`--${h}-bottom`,Math.max(0,Math.min(t,n))+"px")}}function c(){setTimeout(()=>{u=!1},0),a.style.transition="",a.style.cursor="pointer",document.removeEventListener("mousemove",i),document.removeEventListener("touchmove",i),document.removeEventListener("mouseup",c),document.removeEventListener("touchend",c)}a.addEventListener("mousedown",o),a.addEventListener("touchstart",o)}}n.style.display="none",document.body.appendChild(n),2048<t.length&&console.error("The URL is too long, please reduce the number of inputs to prevent the bot from failing to load"),window.addEventListener("message",e=>{var t,n;e.origin===o&&(t=document.getElementById(m))&&e.source===t.contentWindow&&("dify-chatbot-iframe-ready"===e.data.type&&t.contentWindow?.postMessage({type:"dify-chatbot-config",payload:{isToggledByButton:!0,isDraggable:!!y.draggable}},o),"dify-chatbot-expand-change"===e.data.type)&&(a=!a,n=document.getElementById(m))&&(a?n.style.cssText="\n    position: absolute;\n    display: flex;\n    flex-direction: column;\n    justify-content: space-between;\n    top: unset;\n    right: var(--dify-chatbot-bubble-button-right, 1rem); /* Align with dify-chatbot-bubble-button. */\n    bottom: var(--dify-chatbot-bubble-button-bottom, 1rem); /* Align with dify-chatbot-bubble-button. */\n    left: unset;\n    min-width: 24rem;\n    width: 48%;\n    max-width: calc(100vw - 2rem);\n    min-height: 43.75rem;\n    height: 88%;\n    max-height: calc(100vh - 6rem);\n    border: none;\n    z-index: 2147483640;\n    overflow: hidden;\n    user-select: none;\n    transition-property: width, height;\n    transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);\n    transition-duration: 150ms;\n  ":n.style.cssText=l,d())}),document.getElementById(h)||r()}else console.error(t+" is empty or token is not provided")}function p(e="open"){"open"===e?(document.getElementById("openIcon").style.display="block",document.getElementById("closeIcon").style.display="none"):(document.getElementById("openIcon").style.display="none",document.getElementById("closeIcon").style.display="block")}function b(e){"Escape"===e.key&&(e=document.getElementById(m))&&"none"!==e.style.display&&(e.style.display="none",p("open"))}h,h,document.addEventListener("keydown",b),y?.dynamicScript?e():document.body.onload=e})();
+    `,n.appendChild(e),document.body.appendChild(n),n.addEventListener("click",t),n.addEventListener("touchend",t),y.draggable){var a=n;var l=y.dragAxis||"both";let s,d,t,r;function o(e){u=!1,r=("touchstart"===e.type?(s=e.touches[0].clientX-a.offsetLeft,d=e.touches[0].clientY-a.offsetTop,t=e.touches[0].clientX,e.touches[0]):(s=e.clientX-a.offsetLeft,d=e.clientY-a.offsetTop,t=e.clientX,e)).clientY,document.addEventListener("mousemove",i),document.addEventListener("touchmove",i,{passive:!1}),document.addEventListener("mouseup",c),document.addEventListener("touchend",c),e.preventDefault()}function i(n){var o="touchmove"===n.type?n.touches[0]:n,i=o.clientX-t,o=o.clientY-r;if(u=8<Math.abs(i)||8<Math.abs(o)?!0:u){a.style.transition="none",a.style.cursor="grabbing";i=document.getElementById(m);i&&(i.style.display="none",p("open"));let e,t;t="touchmove"===n.type?(e=n.touches[0].clientX-s,window.innerHeight-n.touches[0].clientY-d):(e=n.clientX-s,window.innerHeight-n.clientY-d);o=a.getBoundingClientRect(),i=window.innerWidth-o.width,n=window.innerHeight-o.height;"x"!==l&&"both"!==l||a.style.setProperty(`--${h}-left`,Math.max(0,Math.min(e,i))+"px"),"y"!==l&&"both"!==l||a.style.setProperty(`--${h}-bottom`,Math.max(0,Math.min(t,n))+"px")}}function c(){setTimeout(()=>{u=!1},0),a.style.transition="",a.style.cursor="pointer",document.removeEventListener("mousemove",i),document.removeEventListener("touchmove",i),document.removeEventListener("mouseup",c),document.removeEventListener("touchend",c)}a.addEventListener("mousedown",o),a.addEventListener("touchstart",o)}}n.style.display="none",document.body.appendChild(n),2048<t.length&&console.error("The URL is too long, please reduce the number of inputs to prevent the bot from failing to load"),window.addEventListener("message",e=>{var t,n;e.origin===o&&(t=document.getElementById(m))&&e.source===t.contentWindow&&("dify-chatbot-iframe-ready"===e.data.type&&t.contentWindow?.postMessage({type:"dify-chatbot-config",payload:{isToggledByButton:!0,isDraggable:!!y.draggable}},o),"dify-chatbot-expand-change"===e.data.type)&&(a=!a,n=document.getElementById(m))&&(a?n.style.cssText="\n    position: absolute;\n    display: flex;\n    flex-direction: column;\n    justify-content: space-between;\n    top: unset;\n    right: var(--dify-chatbot-bubble-button-right, 1rem); /* Align with dify-chatbot-bubble-button. */\n    bottom: var(--dify-chatbot-bubble-button-bottom, 1rem); /* Align with dify-chatbot-bubble-button. */\n    left: unset;\n    min-width: 24rem;\n    width: 48%;\n    max-width: 40rem; /* Match mobile breakpoint*/\n    min-height: 43.75rem;\n    height: 88%;\n    max-height: calc(100vh - 6rem);\n    border: none;\n    z-index: 2147483640;\n    overflow: hidden;\n    user-select: none;\n    transition-property: width, height;\n    transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);\n    transition-duration: 150ms;\n  ":n.style.cssText=l,d())}),document.getElementById(h)||r()}else console.error(t+" is empty or token is not provided")}function p(e="open"){"open"===e?(document.getElementById("openIcon").style.display="block",document.getElementById("closeIcon").style.display="none"):(document.getElementById("openIcon").style.display="none",document.getElementById("closeIcon").style.display="block")}function b(e){"Escape"===e.key&&(e=document.getElementById(m))&&"none"!==e.style.display&&(e.style.display="none",p("open"))}h,h,document.addEventListener("keydown",b),y?.dynamicScript?e():document.body.onload=e})();