ngskcloud
/
gcgj-dify-1.7.0
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

Browse Source
pull/21398/head
twwu 11 months ago
parent 87c15062e6 163bae3aaf
commit bad451d5ec
6 changed files with 97 additions and 12 deletions
Show Stats Download Patch File Download Diff File

2
web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/index.tsx
Unescape Escape View File

@ -21,7 +21,7 @@ const Publisher = () => {
const handleOpenChange = useCallback((newOpen: boolean) => {
if (newOpen)
handleSyncWorkflowDraft()
handleSyncWorkflowDraft(true)
setOpen(newOpen)
}, [handleSyncWorkflowDraft])

21
web/app/components/workflow/hooks/use-workflow-variables.ts
Unescape Escape View File

@ -1,6 +1,6 @@
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import { useStore } from '../store'
import { useWorkflowStore } from '../store'
import { getVarType, toNodeAvailableVars } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import type {
Node,
@ -13,8 +13,7 @@ import { useStoreApi } from 'reactflow'
export const useWorkflowVariables = () => {
const { t } = useTranslation()
const environmentVariables = useStore(s => s.environmentVariables)
const conversationVariables = useStore(s => s.conversationVariables)
const workflowStore = useWorkflowStore()
const getNodeAvailableVars = useCallback(({
parentNode,
@ -31,6 +30,11 @@ export const useWorkflowVariables = () => {
hideEnv?: boolean
hideChatVar?: boolean
}): NodeOutPutVar[] => {
const {
conversationVariables,
environmentVariables,
ragPipelineVariables,
} = workflowStore.getState()
return toNodeAvailableVars({
parentNode,
t,
@ -38,9 +42,10 @@ export const useWorkflowVariables = () => {
isChatMode,
environmentVariables: hideEnv ? [] : environmentVariables,
conversationVariables: (isChatMode && !hideChatVar) ? conversationVariables : [],
ragVariables: ragPipelineVariables,
filterVar,
})
}, [conversationVariables, environmentVariables, t])
}, [t, workflowStore])
const getCurrentVariableType = useCallback(({
parentNode,
@ -59,6 +64,11 @@ export const useWorkflowVariables = () => {
isChatMode: boolean
isConstant?: boolean
}) => {
const {
conversationVariables,
environmentVariables,
ragPipelineVariables,
} = workflowStore.getState()
return getVarType({
parentNode,
valueSelector,
@ -69,8 +79,9 @@ export const useWorkflowVariables = () => {
isConstant,
environmentVariables,
conversationVariables,
ragVariables: ragPipelineVariables,
})
}, [conversationVariables, environmentVariables])
}, [workflowStore])
return {
getNodeAvailableVars,

53
web/app/components/workflow/nodes/_base/components/variable/utils.ts
Unescape Escape View File

@ -22,6 +22,7 @@ import type { StartNodeType } from '@/app/components/workflow/nodes/start/types'
import type { ConversationVariable, EnvironmentVariable, Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types'
import type { VariableAssignerNodeType } from '@/app/components/workflow/nodes/variable-assigner/types'
import type { Field as StructField } from '@/app/components/workflow/nodes/llm/types'
import type { RAGPipelineVariable } from '@/models/pipeline'
import {
HTTP_REQUEST_OUTPUT_STRUCT,
@ -51,6 +52,10 @@ export const isConversationVar = (valueSelector: ValueSelector) => {
return valueSelector[0] === 'conversation'
}
export const isRagVariableVar = (valueSelector: ValueSelector) => {
return valueSelector[0] === 'rag'
}
const inputVarTypeToVarType = (type: InputVarType): VarType => {
return ({
[InputVarType.number]: VarType.number,
@ -170,6 +175,7 @@ const formatItem = (
item: any,
isChatMode: boolean,
filterVar: (payload: Var, selector: ValueSelector) => boolean,
ragVars?: Var[],
): NodeOutPutVar => {
const { id, data } = item
@ -460,7 +466,7 @@ const formatItem = (
}
case BlockEnum.DataSource: {
res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType) || []
res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType, ragVars) || []
break
}
@ -484,6 +490,18 @@ const formatItem = (
}) as Var[]
break
}
case 'rag': {
res.vars = data.ragVariables.map((ragVarialbe: RAGPipelineVariable) => {
return {
variable: `rag.${ragVarialbe.variable}`,
type: inputVarTypeToVarType(ragVarialbe.type as any),
des: ragVarialbe.label,
isRagVariable: true,
}
}) as Var[]
break
}
}
const { error_strategy } = data
@ -564,6 +582,7 @@ export const toNodeOutputVars = (
filterVar = (_payload: Var, _selector: ValueSelector) => true,
environmentVariables: EnvironmentVariable[] = [],
conversationVariables: ConversationVariable[] = [],
ragVariables: RAGPipelineVariable[] = [],
): NodeOutPutVar[] => {
// ENV_NODE data format
const ENV_NODE = {
@ -583,6 +602,15 @@ export const toNodeOutputVars = (
chatVarList: conversationVariables,
},
}
// RAG_PIPELINE_NODE data format
const RAG_PIPELINE_NODE = {
id: 'rag',
data: {
title: 'SHARED INPUTS',
type: 'rag',
ragVariables: ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === 'shared'),
},
}
// Sort nodes in reverse chronological order (most recent first)
const sortedNodes = [...nodes].sort((a, b) => {
if (a.data.type === BlockEnum.Start) return 1
@ -599,9 +627,20 @@ export const toNodeOutputVars = (
...sortedNodes.filter(node => SUPPORT_OUTPUT_VARS_NODE.includes(node?.data?.type)),
...(environmentVariables.length > 0 ? [ENV_NODE] : []),
...((isChatMode && conversationVariables.length > 0) ? [CHAT_VAR_NODE] : []),
...(RAG_PIPELINE_NODE.data.ragVariables.length > 0 ? [RAG_PIPELINE_NODE] : []),
].map((node) => {
let ragVariablesInDataSource: RAGPipelineVariable[] = []
if (node.data.type === BlockEnum.DataSource)
ragVariablesInDataSource = ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === node.id)
return {
...formatItem(node, isChatMode, filterVar),
...formatItem(node, isChatMode, filterVar, ragVariablesInDataSource.map(
(ragVariable: RAGPipelineVariable) => ({
variable: ragVariable.variable,
type: inputVarTypeToVarType(ragVariable.type as any),
description: ragVariable.label,
isRagVariable: true,
} as Var),
)),
isStartNode: node.data.type === BlockEnum.Start,
}
}).filter(item => item.vars.length > 0)
@ -719,6 +758,7 @@ export const getVarType = ({
isConstant,
environmentVariables = [],
conversationVariables = [],
ragVariables = [],
}: {
valueSelector: ValueSelector
parentNode?: Node | null
@ -729,6 +769,7 @@ export const getVarType = ({
isConstant?: boolean
environmentVariables?: EnvironmentVariable[]
conversationVariables?: ConversationVariable[]
ragVariables?: RAGPipelineVariable[]
}): VarType => {
if (isConstant)
return VarType.string
@ -739,6 +780,7 @@ export const getVarType = ({
undefined,
environmentVariables,
conversationVariables,
ragVariables,
)
const isIterationInnerVar = parentNode?.data.type === BlockEnum.Iteration
@ -782,6 +824,7 @@ export const getVarType = ({
const isSystem = isSystemVar(valueSelector)
const isEnv = isENV(valueSelector)
const isChatVar = isConversationVar(valueSelector)
const isRagVariable = isRagVariableVar(valueSelector)
const startNode = availableNodes.find((node: any) => {
return node?.data.type === BlockEnum.Start
})
@ -795,7 +838,7 @@ export const getVarType = ({
let type: VarType = VarType.string
let curr: any = targetVar.vars
if (isSystem || isEnv || isChatVar) {
if (isSystem || isEnv || isChatVar || isRagVariable) {
return curr.find((v: any) => v.variable === (valueSelector as ValueSelector).join('.'))?.type
}
else {
@ -846,6 +889,7 @@ export const toNodeAvailableVars = ({
isChatMode,
environmentVariables,
conversationVariables,
ragVariables,
filterVar,
}: {
parentNode?: Node | null
@ -857,6 +901,8 @@ export const toNodeAvailableVars = ({
environmentVariables?: EnvironmentVariable[]
// chat var
conversationVariables?: ConversationVariable[]
// rag variables
ragVariables?: RAGPipelineVariable[]
filterVar: (payload: Var, selector: ValueSelector) => boolean
}): NodeOutPutVar[] => {
const beforeNodesOutputVars = toNodeOutputVars(
@ -865,6 +911,7 @@ export const toNodeAvailableVars = ({
filterVar,
environmentVariables,
conversationVariables,
ragVariables,
)
const isInIteration = parentNode?.data.type === BlockEnum.Iteration
if (isInIteration) {

5
web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx
Unescape Escape View File

@ -23,6 +23,7 @@ import type { Field } from '@/app/components/workflow/nodes/llm/types'
import { FILE_STRUCT } from '@/app/components/workflow/constants'
import { Loop } from '@/app/components/base/icons/src/vender/workflow'
import { noop } from 'lodash-es'
import { InputField } from '@/app/components/base/icons/src/vender/pipeline'
type ObjectChildrenProps = {
nodeId: string
@ -67,6 +68,7 @@ const Item: FC<ItemProps> = ({
const isSys = itemData.variable.startsWith('sys.')
const isEnv = itemData.variable.startsWith('env.')
const isChatVar = itemData.variable.startsWith('conversation.')
const isRagVariable = itemData.isRagVariable
const objStructuredOutput: StructuredOutput | null = useMemo(() => {
if (!isObj) return null
@ -148,10 +150,11 @@ const Item: FC<ItemProps> = ({
onMouseDown={e => e.preventDefault()}
>
<div className='flex w-0 grow items-center'>
{!isEnv && !isChatVar && !isLoopVar && <Variable02 className={cn('h-3.5 w-3.5 shrink-0 text-text-accent', isException && 'text-text-warning')} />}
{!isEnv && !isChatVar && !isLoopVar && !isRagVariable && <Variable02 className={cn('h-3.5 w-3.5 shrink-0 text-text-accent', isException && 'text-text-warning')} />}
{isEnv && <Env className='h-3.5 w-3.5 shrink-0 text-util-colors-violet-violet-600' />}
{isChatVar && <BubbleX className='h-3.5 w-3.5 shrink-0 text-util-colors-teal-teal-700' />}
{isLoopVar && <Loop className='h-3.5 w-3.5 shrink-0 text-util-colors-cyan-cyan-500' />}
{isRagVariable && <InputField className='h-3.5 w-3.5 shrink-0 text-text-accent' />}
{!isEnv && !isChatVar && (
<div title={itemData.variable} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable}</div>
)}

25
web/app/components/workflow/nodes/data-source/default.ts
Unescape Escape View File

@ -11,6 +11,28 @@ const metaData = genNodeMetaData({
const nodeDefault: NodeDefault<DataSourceNodeType> = {
metaData,
defaultValue: {
fileExtensions: [
'txt',
'markdown',
'mdx',
'pdf',
'html',
'xlsx',
'xls',
'vtt',
'properties',
'doc',
'docx',
'csv',
'eml',
'msg',
'pptx',
'xml',
'epub',
'ppt',
'md',
'html',
],
datasource_parameters: {},
datasource_configurations: {},
},
@ -20,7 +42,7 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
errorMessage: '',
}
},
getOutputVars(payload) {
getOutputVars(payload, ragVars = []) {
const {
provider_type,
} = payload
@ -40,6 +62,7 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
]
: []
),
...ragVars,
]
},
}

3
web/app/components/workflow/types.ts
Unescape Escape View File

@ -288,6 +288,7 @@ export type Var = {
isException?: boolean
isLoopVariable?: boolean
nodeId?: string
isRagVariable?: boolean
}
export type NodeOutPutVar = {
@ -313,7 +314,7 @@ export type NodeDefault<T = {}> = {
}
defaultValue: Partial<T>
checkValid: (payload: T, t: any, moreDataForCheckValid?: any) => { isValid: boolean; errorMessage?: string }
getOutputVars?: (payload: T) => Var[]
getOutputVars?: (payload: T, ragVariables?: Var[]) => Var[]
}
export type OnSelectBlock = (type: BlockEnum, toolDefaultValue?: ToolDefaultValue | DataSourceDefaultValue) => void

Write Preview
Loading…
Cancel
Save