Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/index.tsx

@@ -21,7 +21,7 @@ const Publisher = () => {
 
   const handleOpenChange = useCallback((newOpen: boolean) => {
     if (newOpen)
-      handleSyncWorkflowDraft()
+      handleSyncWorkflowDraft(true)
     setOpen(newOpen)
   }, [handleSyncWorkflowDraft])
 

web/app/components/workflow/hooks/use-workflow-variables.ts

@@ -1,6 +1,6 @@
 import { useCallback } from 'react'
 import { useTranslation } from 'react-i18next'
-import { useStore } from '../store'
+import { useWorkflowStore } from '../store'
 import { getVarType, toNodeAvailableVars } from '@/app/components/workflow/nodes/_base/components/variable/utils'
 import type {
   Node,
@@ -13,8 +13,7 @@ import { useStoreApi } from 'reactflow'
 
 export const useWorkflowVariables = () => {
   const { t } = useTranslation()
-  const environmentVariables = useStore(s => s.environmentVariables)
+  const workflowStore = useWorkflowStore()
-  const conversationVariables = useStore(s => s.conversationVariables)
 
   const getNodeAvailableVars = useCallback(({
     parentNode,
@@ -31,6 +30,11 @@ export const useWorkflowVariables = () => {
     hideEnv?: boolean
     hideChatVar?: boolean
   }): NodeOutPutVar[] => {
+    const {
+      conversationVariables,
+      environmentVariables,
+      ragPipelineVariables,
+    } = workflowStore.getState()
     return toNodeAvailableVars({
       parentNode,
       t,
@@ -38,9 +42,10 @@ export const useWorkflowVariables = () => {
       isChatMode,
       environmentVariables: hideEnv ? [] : environmentVariables,
       conversationVariables: (isChatMode && !hideChatVar) ? conversationVariables : [],
+      ragVariables: ragPipelineVariables,
       filterVar,
     })
-  }, [conversationVariables, environmentVariables, t])
+  }, [t, workflowStore])
 
   const getCurrentVariableType = useCallback(({
     parentNode,
@@ -59,6 +64,11 @@ export const useWorkflowVariables = () => {
     isChatMode: boolean
     isConstant?: boolean
   }) => {
+    const {
+      conversationVariables,
+      environmentVariables,
+      ragPipelineVariables,
+    } = workflowStore.getState()
     return getVarType({
       parentNode,
       valueSelector,
@@ -69,8 +79,9 @@ export const useWorkflowVariables = () => {
       isConstant,
       environmentVariables,
       conversationVariables,
+      ragVariables: ragPipelineVariables,
     })
-  }, [conversationVariables, environmentVariables])
+  }, [workflowStore])
 
   return {
     getNodeAvailableVars,

web/app/components/workflow/nodes/_base/components/variable/utils.ts

@@ -22,6 +22,7 @@ import type { StartNodeType } from '@/app/components/workflow/nodes/start/types'
 import type { ConversationVariable, EnvironmentVariable, Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types'
 import type { VariableAssignerNodeType } from '@/app/components/workflow/nodes/variable-assigner/types'
 import type { Field as StructField } from '@/app/components/workflow/nodes/llm/types'
+import type { RAGPipelineVariable } from '@/models/pipeline'
 
 import {
   HTTP_REQUEST_OUTPUT_STRUCT,
@@ -51,6 +52,10 @@ export const isConversationVar = (valueSelector: ValueSelector) => {
   return valueSelector[0] === 'conversation'
 }
 
+export const isRagVariableVar = (valueSelector: ValueSelector) => {
+  return valueSelector[0] === 'rag'
+}
+
 const inputVarTypeToVarType = (type: InputVarType): VarType => {
   return ({
     [InputVarType.number]: VarType.number,
@@ -170,6 +175,7 @@ const formatItem = (
   item: any,
   isChatMode: boolean,
   filterVar: (payload: Var, selector: ValueSelector) => boolean,
+  ragVars?: Var[],
 ): NodeOutPutVar => {
   const { id, data } = item
 
@@ -460,7 +466,7 @@ const formatItem = (
     }
 
     case BlockEnum.DataSource: {
-      res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType) || []
+      res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType, ragVars) || []
       break
     }
 
@@ -484,6 +490,18 @@ const formatItem = (
       }) as Var[]
       break
     }
+
+    case 'rag': {
+      res.vars = data.ragVariables.map((ragVarialbe: RAGPipelineVariable) => {
+        return {
+          variable: `rag.${ragVarialbe.variable}`,
+          type: inputVarTypeToVarType(ragVarialbe.type as any),
+          des: ragVarialbe.label,
+          isRagVariable: true,
+        }
+      }) as Var[]
+      break
+    }
   }
 
   const { error_strategy } = data
@@ -564,6 +582,7 @@ export const toNodeOutputVars = (
   filterVar = (_payload: Var, _selector: ValueSelector) => true,
   environmentVariables: EnvironmentVariable[] = [],
   conversationVariables: ConversationVariable[] = [],
+  ragVariables: RAGPipelineVariable[] = [],
 ): NodeOutPutVar[] => {
   // ENV_NODE data format
   const ENV_NODE = {
@@ -583,6 +602,15 @@ export const toNodeOutputVars = (
       chatVarList: conversationVariables,
     },
   }
+  // RAG_PIPELINE_NODE data format
+  const RAG_PIPELINE_NODE = {
+    id: 'rag',
+    data: {
+      title: 'SHARED INPUTS',
+      type: 'rag',
+      ragVariables: ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === 'shared'),
+    },
+  }
   // Sort nodes in reverse chronological order (most recent first)
   const sortedNodes = [...nodes].sort((a, b) => {
     if (a.data.type === BlockEnum.Start) return 1
@@ -599,9 +627,20 @@ export const toNodeOutputVars = (
     ...sortedNodes.filter(node => SUPPORT_OUTPUT_VARS_NODE.includes(node?.data?.type)),
     ...(environmentVariables.length > 0 ? [ENV_NODE] : []),
     ...((isChatMode && conversationVariables.length > 0) ? [CHAT_VAR_NODE] : []),
+    ...(RAG_PIPELINE_NODE.data.ragVariables.length > 0 ? [RAG_PIPELINE_NODE] : []),
   ].map((node) => {
+    let ragVariablesInDataSource: RAGPipelineVariable[] = []
+    if (node.data.type === BlockEnum.DataSource)
+      ragVariablesInDataSource = ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === node.id)
     return {
-      ...formatItem(node, isChatMode, filterVar),
+      ...formatItem(node, isChatMode, filterVar, ragVariablesInDataSource.map(
+        (ragVariable: RAGPipelineVariable) => ({
+          variable: ragVariable.variable,
+          type: inputVarTypeToVarType(ragVariable.type as any),
+          description: ragVariable.label,
+          isRagVariable: true,
+        } as Var),
+      )),
       isStartNode: node.data.type === BlockEnum.Start,
     }
   }).filter(item => item.vars.length > 0)
@@ -719,6 +758,7 @@ export const getVarType = ({
   isConstant,
   environmentVariables = [],
   conversationVariables = [],
+  ragVariables = [],
 }: {
   valueSelector: ValueSelector
   parentNode?: Node | null
@@ -729,6 +769,7 @@ export const getVarType = ({
   isConstant?: boolean
   environmentVariables?: EnvironmentVariable[]
   conversationVariables?: ConversationVariable[]
+  ragVariables?: RAGPipelineVariable[]
 }): VarType => {
   if (isConstant)
     return VarType.string
@@ -739,6 +780,7 @@ export const getVarType = ({
     undefined,
     environmentVariables,
     conversationVariables,
+    ragVariables,
   )
 
   const isIterationInnerVar = parentNode?.data.type === BlockEnum.Iteration
@@ -782,6 +824,7 @@ export const getVarType = ({
   const isSystem = isSystemVar(valueSelector)
   const isEnv = isENV(valueSelector)
   const isChatVar = isConversationVar(valueSelector)
+  const isRagVariable = isRagVariableVar(valueSelector)
   const startNode = availableNodes.find((node: any) => {
     return node?.data.type === BlockEnum.Start
   })
@@ -795,7 +838,7 @@ export const getVarType = ({
   let type: VarType = VarType.string
   let curr: any = targetVar.vars
 
-  if (isSystem || isEnv || isChatVar) {
+  if (isSystem || isEnv || isChatVar || isRagVariable) {
     return curr.find((v: any) => v.variable === (valueSelector as ValueSelector).join('.'))?.type
   }
   else {
@@ -846,6 +889,7 @@ export const toNodeAvailableVars = ({
   isChatMode,
   environmentVariables,
   conversationVariables,
+  ragVariables,
   filterVar,
 }: {
   parentNode?: Node | null
@@ -857,6 +901,8 @@ export const toNodeAvailableVars = ({
   environmentVariables?: EnvironmentVariable[]
   // chat var
   conversationVariables?: ConversationVariable[]
+  // rag variables
+  ragVariables?: RAGPipelineVariable[]
   filterVar: (payload: Var, selector: ValueSelector) => boolean
 }): NodeOutPutVar[] => {
   const beforeNodesOutputVars = toNodeOutputVars(
@@ -865,6 +911,7 @@ export const toNodeAvailableVars = ({
     filterVar,
     environmentVariables,
     conversationVariables,
+    ragVariables,
   )
   const isInIteration = parentNode?.data.type === BlockEnum.Iteration
   if (isInIteration) {

web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx

@@ -23,6 +23,7 @@ import type { Field } from '@/app/components/workflow/nodes/llm/types'
 import { FILE_STRUCT } from '@/app/components/workflow/constants'
 import { Loop } from '@/app/components/base/icons/src/vender/workflow'
 import { noop } from 'lodash-es'
+import { InputField } from '@/app/components/base/icons/src/vender/pipeline'
 
 type ObjectChildrenProps = {
   nodeId: string
@@ -67,6 +68,7 @@ const Item: FC<ItemProps> = ({
   const isSys = itemData.variable.startsWith('sys.')
   const isEnv = itemData.variable.startsWith('env.')
   const isChatVar = itemData.variable.startsWith('conversation.')
+  const isRagVariable = itemData.isRagVariable
 
   const objStructuredOutput: StructuredOutput | null = useMemo(() => {
     if (!isObj) return null
@@ -148,10 +150,11 @@ const Item: FC<ItemProps> = ({
           onMouseDown={e => e.preventDefault()}
         >
           <div className='flex w-0 grow items-center'>
-            {!isEnv && !isChatVar && !isLoopVar && <Variable02 className={cn('h-3.5 w-3.5 shrink-0 text-text-accent', isException && 'text-text-warning')} />}
+            {!isEnv && !isChatVar && !isLoopVar && !isRagVariable && <Variable02 className={cn('h-3.5 w-3.5 shrink-0 text-text-accent', isException && 'text-text-warning')} />}
             {isEnv && <Env className='h-3.5 w-3.5 shrink-0 text-util-colors-violet-violet-600' />}
             {isChatVar && <BubbleX className='h-3.5 w-3.5 shrink-0 text-util-colors-teal-teal-700' />}
             {isLoopVar && <Loop className='h-3.5 w-3.5 shrink-0 text-util-colors-cyan-cyan-500' />}
+            {isRagVariable && <InputField className='h-3.5 w-3.5 shrink-0 text-text-accent' />}
             {!isEnv && !isChatVar && (
               <div title={itemData.variable} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable}</div>
             )}

web/app/components/workflow/nodes/data-source/default.ts

@@ -11,6 +11,28 @@ const metaData = genNodeMetaData({
 const nodeDefault: NodeDefault<DataSourceNodeType> = {
   metaData,
   defaultValue: {
+    fileExtensions: [
+      'txt',
+      'markdown',
+      'mdx',
+      'pdf',
+      'html',
+      'xlsx',
+      'xls',
+      'vtt',
+      'properties',
+      'doc',
+      'docx',
+      'csv',
+      'eml',
+      'msg',
+      'pptx',
+      'xml',
+      'epub',
+      'ppt',
+      'md',
+      'html',
+    ],
     datasource_parameters: {},
     datasource_configurations: {},
   },
@@ -20,7 +42,7 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
       errorMessage: '',
     }
   },
-  getOutputVars(payload) {
+  getOutputVars(payload, ragVars = []) {
     const {
       provider_type,
     } = payload
@@ -40,6 +62,7 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
           ]
           : []
       ),
+      ...ragVars,
     ]
   },
 }

web/app/components/workflow/types.ts

@@ -288,6 +288,7 @@ export type Var = {
   isException?: boolean
   isLoopVariable?: boolean
   nodeId?: string
+  isRagVariable?: boolean
 }
 
 export type NodeOutPutVar = {
@@ -313,7 +314,7 @@ export type NodeDefault<T = {}> = {
   }
   defaultValue: Partial<T>
   checkValid: (payload: T, t: any, moreDataForCheckValid?: any) => { isValid: boolean; errorMessage?: string }
-  getOutputVars?: (payload: T) => Var[]
+  getOutputVars?: (payload: T, ragVariables?: Var[]) => Var[]
 }
 
 export type OnSelectBlock = (type: BlockEnum, toolDefaultValue?: ToolDefaultValue | DataSourceDefaultValue) => void