pipeline sync draft

web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts

@@ -7,11 +7,15 @@ import {
 import {
   useNodesReadOnly,
 } from '@/app/components/workflow/hooks/use-workflow'
+import { API_PREFIX } from '@/config'
+import { syncWorkflowDraft } from '@/service/workflow'
+import { usePipelineRefreshDraft } from '.'
 
 export const useNodesSyncDraft = () => {
   const store = useStoreApi()
   const workflowStore = useWorkflowStore()
   const { getNodesReadOnly } = useNodesReadOnly()
+  const { handleRefreshWorkflowDraft } = usePipelineRefreshDraft()
 
   const getPostParams = useCallback(() => {
     const {
@@ -47,7 +51,7 @@ export const useNodesSyncDraft = () => {
         })
       })
       return {
-        url: `/rag/pipeline/${pipelineId}/workflows/draft`,
+        url: `/rag/pipelines/${pipelineId}/workflows/draft`,
         params: {
           graph: {
             nodes: producedNodes,
@@ -67,18 +71,57 @@ export const useNodesSyncDraft = () => {
   }, [store, workflowStore])
 
   const syncWorkflowDraftWhenPageClose = useCallback(() => {
-    return true
-  }, [])
-
-  const doSyncWorkflowDraft = useCallback(async () => {
     if (getNodesReadOnly())
       return
     const postParams = getPostParams()
 
-    if (postParams)
+    if (postParams) {
-      return true
+      navigator.sendBeacon(
+        `${API_PREFIX}${postParams.url}?_token=${localStorage.getItem('console_token')}`,
+        JSON.stringify(postParams.params),
+      )
+    }
   }, [getPostParams, getNodesReadOnly])
 
+  const doSyncWorkflowDraft = useCallback(async (
+    notRefreshWhenSyncError?: boolean,
+    callback?: {
+      onSuccess?: () => void
+      onError?: () => void
+      onSettled?: () => void
+    },
+  ) => {
+    if (getNodesReadOnly())
+      return
+    const postParams = getPostParams()
+
+    if (postParams) {
+      const {
+        setSyncWorkflowDraftHash,
+        setDraftUpdatedAt,
+      } = workflowStore.getState()
+      try {
+        console.log('xxx')
+        const res = await syncWorkflowDraft(postParams)
+        setSyncWorkflowDraftHash(res.hash)
+        setDraftUpdatedAt(res.updated_at)
+        callback?.onSuccess && callback.onSuccess()
+      }
+      catch (error: any) {
+        if (error && error.json && !error.bodyUsed) {
+          error.json().then((err: any) => {
+            if (err.code === 'draft_workflow_not_sync' && !notRefreshWhenSyncError)
+              handleRefreshWorkflowDraft()
+          })
+        }
+        callback?.onError && callback.onError()
+      }
+      finally {
+        callback?.onSettled && callback.onSettled()
+      }
+    }
+  }, [getPostParams, getNodesReadOnly, workflowStore, handleRefreshWorkflowDraft])
+
   return {
     doSyncWorkflowDraft,
     syncWorkflowDraftWhenPageClose,

web/app/components/rag-pipeline/hooks/use-pipeline-config.ts

@@ -16,7 +16,7 @@ export const usePipelineConfig = () => {
     setWorkflowConfig(config)
   }, [workflowStore])
   useWorkflowConfig(
-    pipelineId ? `/rag/pipeline/${pipelineId}/workflows/draft/config` : '',
+    pipelineId ? `/rag/pipelines/${pipelineId}/workflows/draft/config` : '',
     handleUpdateWorkflowConfig,
   )
 
@@ -26,7 +26,7 @@ export const usePipelineConfig = () => {
     setNodesDefaultConfigs!(nodesDefaultConfigs)
   }, [workflowStore])
   useWorkflowConfig(
-    pipelineId ? `/rag/pipeline/${pipelineId}/workflows/default-workflow-block-configs` : '',
+    pipelineId ? `/rag/pipelines/${pipelineId}/workflows/default-workflow-block-configs` : '',
     handleUpdateNodesDefaultConfigs,
   )
 
@@ -36,7 +36,7 @@ export const usePipelineConfig = () => {
     setPublishedAt(publishedWorkflow?.created_at)
   }, [workflowStore])
   useWorkflowConfig(
-    pipelineId ? `/rag/pipeline/${pipelineId}/workflows/publish` : '',
+    pipelineId ? `/rag/pipelines/${pipelineId}/workflows/publish` : '',
     handleUpdatePublishedAt,
   )
 }

web/app/components/rag-pipeline/hooks/use-pipeline-init.ts

@@ -3,7 +3,6 @@ import {
   useEffect,
   useState,
 } from 'react'
-import { useParams } from 'next/navigation'
 import {
   useWorkflowStore,
 } from '@/app/components/workflow/store'
@@ -13,10 +12,10 @@ import {
   syncWorkflowDraft,
 } from '@/service/workflow'
 import type { FetchWorkflowDraftResponse } from '@/types/workflow'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { usePipelineConfig } from './use-pipeline-config'
 
 export const usePipelineInit = () => {
-  const { datasetId } = useParams()
   const workflowStore = useWorkflowStore()
   const {
     nodes: nodesTemplate,
@@ -24,9 +23,10 @@ export const usePipelineInit = () => {
   } = usePipelineTemplate()
   const [data, setData] = useState<FetchWorkflowDraftResponse>()
   const [isLoading, setIsLoading] = useState(true)
+  const datasetId = useDatasetDetailContextWithSelector(s => s.dataset)?.pipeline_id
 
   useEffect(() => {
-    workflowStore.setState({ pipelineId: datasetId as string })
+    workflowStore.setState({ pipelineId: datasetId })
   }, [datasetId, workflowStore])
 
   usePipelineConfig()
@@ -41,7 +41,7 @@ export const usePipelineInit = () => {
       setRagPipelineVariables,
     } = workflowStore.getState()
     try {
-      const res = await fetchWorkflowDraft(`/rag/pipeline/${datasetId}/workflows/draft`)
+      const res = await fetchWorkflowDraft(`/rag/pipelines/${datasetId}/workflows/draft`)
       setData(res)
       setDraftUpdatedAt(res.updated_at)
       setToolPublished(res.tool_published)
@@ -60,7 +60,7 @@ export const usePipelineInit = () => {
           if (err.code === 'draft_workflow_not_exist') {
             workflowStore.setState({ notInitialWorkflow: true })
             syncWorkflowDraft({
-              url: `/rag/pipeline/${datasetId}/workflows/draft`,
+              url: `/rag/pipelines/${datasetId}/workflows/draft`,
               params: {
                 graph: {
                   nodes: nodesTemplate,

web/app/components/rag-pipeline/hooks/use-pipeline-refresh-draft.ts

@@ -1,9 +1,32 @@
 import { useCallback } from 'react'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+import { fetchWorkflowDraft } from '@/service/workflow'
+import type { WorkflowDataUpdater } from '@/app/components/workflow/types'
+import { useWorkflowUpdate } from '@/app/components/workflow/hooks'
 
 export const usePipelineRefreshDraft = () => {
+  const workflowStore = useWorkflowStore()
+  const { handleUpdateWorkflowCanvas } = useWorkflowUpdate()
+
   const handleRefreshWorkflowDraft = useCallback(() => {
-    return true
+    const {
-  }, [])
+      pipelineId,
+      setSyncWorkflowDraftHash,
+      setIsSyncingWorkflowDraft,
+      setEnvironmentVariables,
+      setEnvSecrets,
+    } = workflowStore.getState()
+    setIsSyncingWorkflowDraft(true)
+    fetchWorkflowDraft(`/rag/pipelines/${pipelineId}/workflows/draft`).then((response) => {
+      handleUpdateWorkflowCanvas(response.graph as WorkflowDataUpdater)
+      setSyncWorkflowDraftHash(response.hash)
+      setEnvSecrets((response.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
+        acc[env.id] = env.value
+        return acc
+      }, {} as Record<string, string>))
+      setEnvironmentVariables(response.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [])
+    }).finally(() => setIsSyncingWorkflowDraft(false))
+  }, [handleUpdateWorkflowCanvas, workflowStore])
 
   return {
     handleRefreshWorkflowDraft,

web/app/components/rag-pipeline/hooks/use-pipeline-run.ts

@@ -141,8 +141,6 @@ export const usePipelineRun = () => {
       resultText: '',
     })
 
-    return true
-
     ssePost(
       url,
       {
@@ -279,7 +277,6 @@ export const usePipelineRun = () => {
   const handleStopRun = useCallback((taskId: string) => {
     const { pipelineId } = workflowStore.getState()
 
-    return
     stopWorkflowRun(`/rag/pipeline/${pipelineId}/workflow-runs/tasks/${taskId}/stop`)
   }, [workflowStore])
 

web/app/components/rag-pipeline/index.tsx

@@ -1,49 +1,64 @@
+import { useMemo } from 'react'
 import WorkflowWithDefaultContext from '@/app/components/workflow'
 import {
   WorkflowContextProvider,
 } from '@/app/components/workflow/context'
 import type { InjectWorkflowStoreSliceFn } from '@/app/components/workflow/store'
-import { generateNewNode } from '@/app/components/workflow/utils'
-import knowledgeBaseNodeDefault from '@/app/components/workflow/nodes/knowledge-base/default'
 import {
-  NODE_WIDTH_X_OFFSET,
+  initialEdges,
-  START_INITIAL_POSITION,
+  initialNodes,
-} from '@/app/components/workflow/constants'
+} from '@/app/components/workflow/utils'
+import Loading from '@/app/components/base/loading'
 import { createRagPipelineSliceSlice } from './store'
 import RagPipelineMain from './components/rag-pipeline-main'
-// import { usePipelineInit } from './hooks'
+import { usePipelineInit } from './hooks'
 
 const RagPipeline = () => {
-  // const {
+  const {
-  //   data,
+    data,
-  //   isLoading,
+    isLoading,
-  // } = usePipelineInit()
+  } = usePipelineInit()
-  const { newNode: knowledgeBaseNode } = generateNewNode({
+  const nodesData = useMemo(() => {
-    data: {
+    if (data)
-      type: knowledgeBaseNodeDefault.metaData.type,
+      return initialNodes(data.graph.nodes, data.graph.edges)
-      title: 'knowledge-base',
+
-      ...knowledgeBaseNodeDefault.defaultValue,
+    return []
-    },
+  }, [data])
-    position: {
+  const edgesData = useMemo(() => {
-      x: START_INITIAL_POSITION.x + NODE_WIDTH_X_OFFSET,
+    if (data)
-      y: START_INITIAL_POSITION.y,
+      return initialEdges(data.graph.edges, data.graph.nodes)
-    },
+
-  } as any)
+    return []
+  }, [data])
+
+  if (!data || isLoading) {
+    return (
+      <div className='relative flex h-full w-full items-center justify-center'>
+        <Loading />
+      </div>
+    )
+  }
   return (
-    <WorkflowContextProvider
-      injectWorkflowStoreSliceFn={createRagPipelineSliceSlice as InjectWorkflowStoreSliceFn}
-    >
     <WorkflowWithDefaultContext
-        edges={[]}
+      edges={edgesData}
-        nodes={[knowledgeBaseNode]}
+      nodes={nodesData}
     >
       <RagPipelineMain
-          edges={[]}
+        edges={edgesData}
-          nodes={[knowledgeBaseNode]}
+        nodes={nodesData}
       />
     </WorkflowWithDefaultContext>
+  )
+}
+
+const RagPipelineWrapper = () => {
+  return (
+    <WorkflowContextProvider
+      injectWorkflowStoreSliceFn={createRagPipelineSliceSlice as InjectWorkflowStoreSliceFn}
+    >
+      <RagPipeline />
     </WorkflowContextProvider>
   )
 }
 
-export default RagPipeline
+export default RagPipelineWrapper

web/app/components/workflow/block-selector/index.tsx

@@ -32,7 +32,7 @@ const NodeSelectorWrapper = (props: NodeSelectorProps) => {
   }, [availableNodesMetaData?.nodes])
 
   const pipelineId = useStore(s => s.pipelineId)
-  const { data: dataSourceList } = useDataSourceList(!pipelineId)
+  const { data: dataSourceList } = useDataSourceList(!!pipelineId)
 
   return (
     <NodeSelector

web/models/datasets.ts

@@ -80,6 +80,7 @@ export type DataSet = {
   built_in_field_enabled: boolean
   doc_metadata?: MetadataInDoc[]
   keyword_number?: number
+  pipeline_id?: string
 }
 
 export type ExternalAPIItem = {