Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

api/migrations/versions/2024_10_10_0516-bbadea11becb_add_name_and_size_to_tool_files.py

@@ -5,45 +5,61 @@ Revises: 33f5fac87f29
 Create Date: 2024-10-10 05:16:14.764268
 
 """
-from alembic import op
+
-import models as models
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from alembic import op, context
 
 # revision identifiers, used by Alembic.
-revision = 'bbadea11becb'
+revision = "bbadea11becb"
-down_revision = 'd8e744d88ed6'
+down_revision = "d8e744d88ed6"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
+    def _has_name_or_size_column() -> bool:
+        # We cannot access the database in offline mode, so assume
+        # the "name" and "size" columns do not exist.
+        if context.is_offline_mode():
+            # Log a warning message to inform the user that the database schema cannot be inspected
+            # in offline mode, and the generated SQL may not accurately reflect the actual execution.
+            op.execute(
+                "-- Executing in offline mode, assuming the name and size columns do not exist.\n"
+                "-- The generated SQL may differ from what will actually be executed.\n"
+                "-- Please review the migration script carefully!"
+            )
+
+            return False
+        # Use SQLAlchemy inspector to get the columns of the 'tool_files' table
+        inspector = sa.inspect(conn)
+        columns = [col["name"] for col in inspector.get_columns("tool_files")]
+
+        # If 'name' or 'size' columns already exist, exit the upgrade function
+        if "name" in columns or "size" in columns:
+            return True
+        return False
+
     # ### commands auto generated by Alembic - please adjust! ###
     # Get the database connection
     conn = op.get_bind()
 
-    # Use SQLAlchemy inspector to get the columns of the 'tool_files' table
+    if _has_name_or_size_column():
-    inspector = sa.inspect(conn)
-    columns = [col['name'] for col in inspector.get_columns('tool_files')]
-
-    # If 'name' or 'size' columns already exist, exit the upgrade function
-    if 'name' in columns or 'size' in columns:
         return
 
-    with op.batch_alter_table('tool_files', schema=None) as batch_op:
+    with op.batch_alter_table("tool_files", schema=None) as batch_op:
-        batch_op.add_column(sa.Column('name', sa.String(), nullable=True))
+        batch_op.add_column(sa.Column("name", sa.String(), nullable=True))
-        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column("size", sa.Integer(), nullable=True))
     op.execute("UPDATE tool_files SET name = '' WHERE name IS NULL")
     op.execute("UPDATE tool_files SET size = -1 WHERE size IS NULL")
-    with op.batch_alter_table('tool_files', schema=None) as batch_op:
+    with op.batch_alter_table("tool_files", schema=None) as batch_op:
-        batch_op.alter_column('name', existing_type=sa.String(), nullable=False)
+        batch_op.alter_column("name", existing_type=sa.String(), nullable=False)
-        batch_op.alter_column('size', existing_type=sa.Integer(), nullable=False)
+        batch_op.alter_column("size", existing_type=sa.Integer(), nullable=False)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('tool_files', schema=None) as batch_op:
+    with op.batch_alter_table("tool_files", schema=None) as batch_op:
-        batch_op.drop_column('size')
+        batch_op.drop_column("size")
-        batch_op.drop_column('name')
+        batch_op.drop_column("name")
     # ### end Alembic commands ###

api/migrations/versions/2024_12_23_1154-d7999dfa4aae_remove_workflow_node_executions_retry_.py

@@ -5,28 +5,38 @@ Revises: e1944c35e15e
 Create Date: 2024-12-23 11:54:15.344543
 
 """
-from alembic import op
+
-import models as models
+from alembic import op, context
-import sqlalchemy as sa
 from sqlalchemy import inspect
 
 
 # revision identifiers, used by Alembic.
-revision = 'd7999dfa4aae'
+revision = "d7999dfa4aae"
-down_revision = 'e1944c35e15e'
+down_revision = "e1944c35e15e"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    # Check if column exists before attempting to remove it
+    def _has_retry_index_column() -> bool:
-    conn = op.get_bind()
+        if context.is_offline_mode():
-    inspector = inspect(conn)
+            # Log a warning message to inform the user that the database schema cannot be inspected
-    has_column = 'retry_index' in [col['name'] for col in inspector.get_columns('workflow_node_executions')]
+            # in offline mode, and the generated SQL may not accurately reflect the actual execution.
+            op.execute(
+                '-- Executing in offline mode: assuming the "retry_index" column does not exist.\n'
+                "-- The generated SQL may differ from what will actually be executed.\n"
+                "-- Please review the migration script carefully!"
+            )
+            return False
+        conn = op.get_bind()
+        inspector = inspect(conn)
+        return "retry_index" in [col["name"] for col in inspector.get_columns("workflow_node_executions")]
+
+    has_column = _has_retry_index_column()
 
     if has_column:
-        with op.batch_alter_table('workflow_node_executions', schema=None) as batch_op:
+        with op.batch_alter_table("workflow_node_executions", schema=None) as batch_op:
-            batch_op.drop_column('retry_index')
+            batch_op.drop_column("retry_index")
 
 
 def downgrade():

api/models/tools.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-from typing import Any, Optional, cast
+from typing import Any, cast
 
 import sqlalchemy as sa
 from deprecated import deprecated
@@ -304,8 +304,11 @@ class DeprecatedPublishedAppTool(Base):
         db.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"),
     )
 
+    id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))
     # id of the app
     app_id = db.Column(StringUUID, ForeignKey("apps.id"), nullable=False)
+
+    user_id: Mapped[str] = db.Column(StringUUID, nullable=False)
     # who published this tool
     description = db.Column(db.Text, nullable=False)
     # llm_description of the tool, for LLM
@@ -325,34 +328,3 @@ class DeprecatedPublishedAppTool(Base):
     @property
     def description_i18n(self) -> I18nObject:
         return I18nObject(**json.loads(self.description))
-
-    id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))
-    user_id: Mapped[str] = db.Column(StringUUID, nullable=False)
-    tenant_id: Mapped[str] = db.Column(StringUUID, nullable=False)
-    conversation_id: Mapped[Optional[str]] = db.Column(StringUUID, nullable=True)
-    file_key: Mapped[str] = db.Column(db.String(255), nullable=False)
-    mimetype: Mapped[str] = db.Column(db.String(255), nullable=False)
-    original_url: Mapped[Optional[str]] = db.Column(db.String(2048), nullable=True)
-    name: Mapped[str] = mapped_column(default="")
-    size: Mapped[int] = mapped_column(default=-1)
-
-    def __init__(
-        self,
-        *,
-        user_id: str,
-        tenant_id: str,
-        conversation_id: Optional[str] = None,
-        file_key: str,
-        mimetype: str,
-        original_url: Optional[str] = None,
-        name: str,
-        size: int,
-    ):
-        self.user_id = user_id
-        self.tenant_id = tenant_id
-        self.conversation_id = conversation_id
-        self.file_key = file_key
-        self.mimetype = mimetype
-        self.original_url = original_url
-        self.name = name
-        self.size = size

web/app/components/workflow-app/components/workflow-main.tsx

@@ -9,6 +9,7 @@ import WorkflowChildren from './workflow-children'
 import {
   useAvailableNodesMetaData,
   useNodesSyncDraft,
+  useWorkflowRefreshDraft,
   useWorkflowRun,
   useWorkflowStartRun,
 } from '../hooks'
@@ -33,6 +34,7 @@ const WorkflowMain = ({
     doSyncWorkflowDraft,
     syncWorkflowDraftWhenPageClose,
   } = useNodesSyncDraft()
+  const { handleRefreshWorkflowDraft } = useWorkflowRefreshDraft()
   const {
     handleBackupDraft,
     handleLoadBackupDraft,
@@ -51,6 +53,7 @@ const WorkflowMain = ({
     return {
       syncWorkflowDraftWhenPageClose,
       doSyncWorkflowDraft,
+      handleRefreshWorkflowDraft,
       handleBackupDraft,
       handleLoadBackupDraft,
       handleRestoreFromPublishedWorkflow,
@@ -64,6 +67,7 @@ const WorkflowMain = ({
   }, [
     syncWorkflowDraftWhenPageClose,
     doSyncWorkflowDraft,
+    handleRefreshWorkflowDraft,
     handleBackupDraft,
     handleLoadBackupDraft,
     handleRestoreFromPublishedWorkflow,

web/app/components/workflow-app/hooks/index.ts

@@ -5,3 +5,4 @@ export * from './use-workflow-run'
 export * from './use-workflow-start-run'
 export * from './use-is-chat-mode'
 export * from './use-available-nodes-meta-data'
+export * from './use-workflow-refresh-draft'

web/app/components/workflow-app/hooks/use-nodes-sync-draft.ts

@@ -6,20 +6,20 @@ import {
   useWorkflowStore,
 } from '@/app/components/workflow/store'
 import { BlockEnum } from '@/app/components/workflow/types'
-import { useWorkflowUpdate } from '@/app/components/workflow/hooks'
 import {
   useNodesReadOnly,
 } from '@/app/components/workflow/hooks/use-workflow'
 import { syncWorkflowDraft } from '@/service/workflow'
 import { useFeaturesStore } from '@/app/components/base/features/hooks'
 import { API_PREFIX } from '@/config'
+import { useWorkflowRefreshDraft } from '.'
 
 export const useNodesSyncDraft = () => {
   const store = useStoreApi()
   const workflowStore = useWorkflowStore()
   const featuresStore = useFeaturesStore()
   const { getNodesReadOnly } = useNodesReadOnly()
-  const { handleRefreshWorkflowDraft } = useWorkflowUpdate()
+  const { handleRefreshWorkflowDraft } = useWorkflowRefreshDraft()
   const params = useParams()
 
   const getPostParams = useCallback(() => {

web/app/components/workflow-app/hooks/use-workflow-refresh-draft.ts

@@ -0,0 +1,36 @@
+import { useCallback } from 'react'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+import { fetchWorkflowDraft } from '@/service/workflow'
+import type { WorkflowDataUpdater } from '@/app/components/workflow/types'
+import { useWorkflowUpdate } from '@/app/components/workflow/hooks'
+
+export const useWorkflowRefreshDraft = () => {
+  const workflowStore = useWorkflowStore()
+  const { handleUpdateWorkflowCanvas } = useWorkflowUpdate()
+
+  const handleRefreshWorkflowDraft = useCallback(() => {
+    const {
+      appId,
+      setSyncWorkflowDraftHash,
+      setIsSyncingWorkflowDraft,
+      setEnvironmentVariables,
+      setEnvSecrets,
+      setConversationVariables,
+    } = workflowStore.getState()
+    setIsSyncingWorkflowDraft(true)
+    fetchWorkflowDraft(`/apps/${appId}/workflows/draft`).then((response) => {
+      handleUpdateWorkflowCanvas(response.graph as WorkflowDataUpdater)
+      setSyncWorkflowDraftHash(response.hash)
+      setEnvSecrets((response.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
+        acc[env.id] = env.value
+        return acc
+      }, {} as Record<string, string>))
+      setEnvironmentVariables(response.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [])
+      setConversationVariables(response.conversation_variables || [])
+    }).finally(() => setIsSyncingWorkflowDraft(false))
+  }, [handleUpdateWorkflowCanvas, workflowStore])
+
+  return {
+    handleRefreshWorkflowDraft,
+  }
+}

web/app/components/workflow/hooks-store/store.ts

@@ -26,6 +26,7 @@ export type CommonHooksFnMap = {
     }
   ) => Promise<void>
   syncWorkflowDraftWhenPageClose: () => void
+  handleRefreshWorkflowDraft: () => void
   handleBackupDraft: () => void
   handleLoadBackupDraft: () => void
   handleRestoreFromPublishedWorkflow: (...args: any[]) => void
@@ -44,6 +45,7 @@ export type Shape = {
 export const createHooksStore = ({
   doSyncWorkflowDraft = async () => noop(),
   syncWorkflowDraftWhenPageClose = noop,
+  handleRefreshWorkflowDraft = noop,
   handleBackupDraft = noop,
   handleLoadBackupDraft = noop,
   handleRestoreFromPublishedWorkflow = noop,
@@ -60,6 +62,7 @@ export const createHooksStore = ({
     refreshAll: props => set(state => ({ ...state, ...props })),
     doSyncWorkflowDraft,
     syncWorkflowDraftWhenPageClose,
+    handleRefreshWorkflowDraft,
     handleBackupDraft,
     handleLoadBackupDraft,
     handleRestoreFromPublishedWorkflow,

web/app/components/workflow/hooks/index.ts

@@ -17,3 +17,4 @@ export * from './use-workflow-mode'
 export * from './use-format-time-from-now'
 export * from './use-nodes-meta-data'
 export * from './use-available-blocks'
+export * from './use-workflow-refresh-draft'

web/app/components/workflow/hooks/use-workflow-interactions.ts

@@ -313,7 +313,6 @@ export const useWorkflowZoom = () => {
 
 export const useWorkflowUpdate = () => {
   const reactflow = useReactFlow()
-  const workflowStore = useWorkflowStore()
   const { eventEmitter } = useEventEmitterContextContext()
 
   const handleUpdateWorkflowCanvas = useCallback((payload: WorkflowDataUpdater) => {
@@ -333,32 +332,8 @@ export const useWorkflowUpdate = () => {
     setViewport(viewport)
   }, [eventEmitter, reactflow])
 
-  const handleRefreshWorkflowDraft = useCallback(() => {
-    const {
-      appId,
-      setSyncWorkflowDraftHash,
-      setIsSyncingWorkflowDraft,
-      setEnvironmentVariables,
-      setEnvSecrets,
-      setConversationVariables,
-    } = workflowStore.getState()
-    setIsSyncingWorkflowDraft(true)
-    fetchWorkflowDraft(`/apps/${appId}/workflows/draft`).then((response) => {
-      handleUpdateWorkflowCanvas(response.graph as WorkflowDataUpdater)
-      setSyncWorkflowDraftHash(response.hash)
-      setEnvSecrets((response.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
-        acc[env.id] = env.value
-        return acc
-      }, {} as Record<string, string>))
-      setEnvironmentVariables(response.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [])
-      // #TODO chatVar sync#
-      setConversationVariables(response.conversation_variables || [])
-    }).finally(() => setIsSyncingWorkflowDraft(false))
-  }, [handleUpdateWorkflowCanvas, workflowStore])
-
   return {
     handleUpdateWorkflowCanvas,
-    handleRefreshWorkflowDraft,
   }
 }
 

web/app/components/workflow/hooks/use-workflow-refresh-draft.ts

@@ -0,0 +1,9 @@
+import { useHooksStore } from '@/app/components/workflow/hooks-store'
+
+export const useWorkflowRefreshDraft = () => {
+  const handleRefreshWorkflowDraft = useHooksStore(s => s.handleRefreshWorkflowDraft)
+
+  return {
+    handleRefreshWorkflowDraft,
+  }
+}

web/app/components/workflow/index.tsx

@@ -44,7 +44,7 @@ import {
   useShortcuts,
   useWorkflow,
   useWorkflowReadOnly,
-  useWorkflowUpdate,
+  useWorkflowRefreshDraft,
 } from './hooks'
 import CustomNode from './nodes'
 import CustomNoteNode from './note-node'
@@ -160,7 +160,7 @@ export const Workflow: FC<WorkflowProps> = memo(({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
-  const { handleRefreshWorkflowDraft } = useWorkflowUpdate()
+  const { handleRefreshWorkflowDraft } = useWorkflowRefreshDraft()
   const handleSyncWorkflowDraftWhenPageClose = useCallback(() => {
     if (document.visibilityState === 'hidden')
       syncWorkflowDraftWhenPageClose()