feat: update data source handling and improve processing parameters integration

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -274,7 +274,6 @@ const CreateFormPipeline = () => {
                 {datasource?.type === DatasourceType.websiteCrawl && (
                   <WebsiteCrawl
                     nodeId={datasource?.nodeId || ''}
-                    variables={[]} // todo: replace with actual variables if needed
                     headerInfo={{
                       title: datasource.description,
                       docTitle: datasource.docTitle || '',
@@ -284,6 +283,7 @@ const CreateFormPipeline = () => {
                     onCheckedCrawlResultChange={setWebsitePages}
                     onJobIdChange={setWebsiteCrawlJobId}
                     onPreview={updateCurrentWebsite}
+                    usingPublished
                   />
                 )}
                 {isShowVectorSpaceFull && (

web/app/components/rag-pipeline/components/input-field/index.tsx

@@ -56,14 +56,15 @@ const InputFieldDialog = ({
 
   const { doSyncWorkflowDraft } = useNodesSyncDraft()
 
-  useUnmount(() => {
+  const { run: syncWorkflowDraft, cancel: cancelSyncWorkflowDraft } = useDebounceFn(async () => {
-    doSyncWorkflowDraft()
+    await doSyncWorkflowDraft()
+  }, {
+    wait: 500,
   })
 
-  const { run: syncWorkflowDraft } = useDebounceFn(() => {
+  useUnmount(() => {
+    cancelSyncWorkflowDraft()
     doSyncWorkflowDraft()
-  }, {
-    wait: 500,
   })
 
   const datasourceNodeDataMap = useMemo(() => {
@@ -76,7 +77,7 @@ const InputFieldDialog = ({
     return datasourceNodeDataMap
   }, [nodes])
 
-  const updateInputFields = useCallback((key: string, value: InputVar[]) => {
+  const updateInputFields = useCallback(async (key: string, value: InputVar[]) => {
     inputFieldsMap.current[key] = value
     const newRagPipelineVariables: RAGPipelineVariables = []
     Object.keys(inputFieldsMap.current).forEach((key) => {
@@ -89,7 +90,7 @@ const InputFieldDialog = ({
       })
     })
     setRagPipelineVariables?.(newRagPipelineVariables)
-    syncWorkflowDraft()
+    await syncWorkflowDraft()
   }, [setRagPipelineVariables, syncWorkflowDraft])
 
   const closePanel = useCallback(() => {

web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx

@@ -3,17 +3,24 @@ import { useTranslation } from 'react-i18next'
 import DataSourceOptions from '../../panel/test-run/data-source-options'
 import Form from './form'
 import type { Datasource } from '../../panel/test-run/types'
+import { useStore } from '@/app/components/workflow/store'
+import { useDraftPipelinePreProcessingParams } from '@/service/use-pipeline'
 
 type DatasourceProps = {
   onSelect: (dataSource: Datasource) => void
-  datasourceNodeId: string
+  dataSourceNodeId: string
 }
 
 const DataSource = ({
   onSelect: setDatasource,
-  datasourceNodeId,
+  dataSourceNodeId,
 }: DatasourceProps) => {
   const { t } = useTranslation()
+  const pipelineId = useStore(state => state.pipelineId)
+  const { data: paramsConfig } = useDraftPipelinePreProcessingParams({
+    pipeline_id: pipelineId!,
+    node_id: dataSourceNodeId,
+  }, !!pipelineId && !!dataSourceNodeId)
 
   return (
     <div className='flex flex-col'>
@@ -23,10 +30,10 @@ const DataSource = ({
       <div className='px-4 py-2'>
         <DataSourceOptions
           onSelect={setDatasource}
-          datasourceNodeId={datasourceNodeId}
+          dataSourceNodeId={dataSourceNodeId}
         />
       </div>
-      <Form variables={[]} />
+      <Form variables={paramsConfig?.variables || []} />
     </div>
   )
 }

web/app/components/rag-pipeline/components/input-field/preview/index.tsx

@@ -40,13 +40,15 @@ const PreviewPanel = ({
           <RiCloseLine className='size-4 text-text-tertiary' />
         </button>
       </div>
+      {/* Data source form Preview */}
       <DataSource
         onSelect={setDatasource}
-        datasourceNodeId={datasource?.nodeId || ''}
+        dataSourceNodeId={datasource?.nodeId || ''}
       />
       <div className='px-4 py-2'>
         <Divider type='horizontal' className='bg-divider-subtle' />
       </div>
+      {/* Process documents form Preview */}
       <ProcessDocuments dataSourceNodeId={datasource?.nodeId || ''} />
     </DialogWrapper>
   )

web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx

@@ -4,12 +4,12 @@ import OptionCard from './option-card'
 import type { Datasource } from '../types'
 
 type DataSourceOptionsProps = {
-  datasourceNodeId: string
+  dataSourceNodeId: string
   onSelect: (option: Datasource) => void
 }
 
 const DataSourceOptions = ({
-  datasourceNodeId,
+  dataSourceNodeId,
   onSelect,
 }: DataSourceOptionsProps) => {
   const { datasources, options } = useDatasourceOptions()
@@ -34,7 +34,7 @@ const DataSourceOptions = ({
           key={option.value}
           label={option.label}
           nodeData={option.data}
-          selected={datasourceNodeId === option.value}
+          selected={dataSourceNodeId === option.value}
           onClick={handelSelect.bind(null, option.value)}
         />
       ))}

web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx

@@ -1,5 +1,5 @@
 'use client'
-import React, { useCallback, useEffect, useState } from 'react'
+import React, { useCallback, useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import type { CrawlResultItem } from '@/models/datasets'
 import Header from '@/app/components/datasets/create/website/base/header'
@@ -7,15 +7,17 @@ import Options from './options'
 import Crawling from './crawling'
 import ErrorMessage from './error-message'
 import CrawledResult from './crawled-result'
-import type { RAGPipelineVariables } from '@/models/pipeline'
+import {
-import { useDatasourceNodeRun } from '@/service/use-pipeline'
+  useDatasourceNodeRun,
-import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+  useDraftPipelinePreProcessingParams,
+  usePublishedPipelineProcessingParams,
+} from '@/service/use-pipeline'
+import { useStore } from '@/app/components/workflow/store'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 type CrawlerProps = {
   nodeId: string
-  variables: RAGPipelineVariables
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
@@ -25,6 +27,7 @@ type CrawlerProps = {
     docLink: string
   }
   onPreview?: (payload: CrawlResultItem) => void
+  usingPublished?: boolean
 }
 
 enum Step {
@@ -35,17 +38,23 @@ enum Step {
 
 const Crawler = ({
   nodeId,
-  variables,
   checkedCrawlResult,
   headerInfo,
   onCheckedCrawlResultChange,
   onJobIdChange,
   onPreview,
+  usingPublished = false,
 }: CrawlerProps) => {
   const { t } = useTranslation()
   const [step, setStep] = useState<Step>(Step.init)
   const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
-  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const pipelineId = useStore(s => s.pipelineId)
+
+  const usePreProcessingParams = useRef(usingPublished ? usePublishedPipelineProcessingParams : useDraftPipelinePreProcessingParams)
+  const { data: paramsConfig } = usePreProcessingParams.current({
+    pipeline_id: pipelineId!,
+    node_id: nodeId,
+  }, !!pipelineId && !!nodeId)
 
   useEffect(() => {
     if (step !== Step.init)
@@ -95,7 +104,7 @@ const Crawler = ({
       />
       <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
         <Options
-          variables={variables}
+          variables={paramsConfig?.variables || []}
           isRunning={isRunning}
           controlFoldOptions={controlFoldOptions}
           onSubmit={(value) => {

web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx

@@ -1,12 +1,10 @@
 'use client'
 import React from 'react'
 import type { CrawlResultItem } from '@/models/datasets'
-import type { RAGPipelineVariables } from '@/models/pipeline'
 import Crawler from './base/crawler'
 
 type WebsiteCrawlProps = {
   nodeId: string
-  variables: RAGPipelineVariables
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
@@ -16,26 +14,27 @@ type WebsiteCrawlProps = {
     docLink: string
   }
   onPreview?: (payload: CrawlResultItem) => void
+  usingPublished?: boolean
 }
 
 const WebsiteCrawl = ({
   nodeId,
-  variables,
   checkedCrawlResult,
   headerInfo,
   onCheckedCrawlResultChange,
   onJobIdChange,
   onPreview,
+  usingPublished,
 }: WebsiteCrawlProps) => {
   return (
     <Crawler
       nodeId={nodeId}
-      variables={variables}
       checkedCrawlResult={checkedCrawlResult}
       headerInfo={headerInfo}
       onCheckedCrawlResultChange={onCheckedCrawlResultChange}
       onJobIdChange={onJobIdChange}
       onPreview={onPreview}
+      usingPublished={usingPublished}
     />
   )
 }

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -117,7 +117,7 @@ const TestRunPanel = () => {
             <>
               <div className='flex flex-col gap-y-4 px-4 py-2'>
                 <DataSourceOptions
-                  datasourceNodeId={datasource?.nodeId || ''}
+                  dataSourceNodeId={datasource?.nodeId || ''}
                   onSelect={setDatasource}
                 />
                 {datasource?.type === DatasourceType.localFile && (
@@ -139,7 +139,6 @@ const TestRunPanel = () => {
                 {datasource?.type === DatasourceType.websiteCrawl && (
                   <WebsiteCrawl
                     nodeId={datasource?.nodeId || ''}
-                    variables={[]} // todo: replace with actual variables if needed
                     checkedCrawlResult={websitePages}
                     headerInfo={{
                       title: datasource.description,

web/service/use-pipeline.ts

@@ -141,7 +141,7 @@ export const useDatasourceNodeRun = (
 export const useDraftPipelineProcessingParams = (params: PipelineProcessingParamsRequest, enabled = true) => {
   const { pipeline_id, node_id } = params
   return useQuery<PipelineProcessingParamsResponse>({
-    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
+    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id, node_id],
     queryFn: () => {
       return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/processing/parameters`, {
         params: {
@@ -157,7 +157,7 @@ export const useDraftPipelineProcessingParams = (params: PipelineProcessingParam
 export const usePublishedPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
   const { pipeline_id, node_id } = params
   return useQuery<PipelineProcessingParamsResponse>({
-    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
+    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id, node_id],
     queryFn: () => {
       return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
         params: {
@@ -165,6 +165,7 @@ export const usePublishedPipelineProcessingParams = (params: PipelineProcessingP
         },
       })
     },
+    staleTime: 0,
   })
 }
 
@@ -254,7 +255,7 @@ export const useUpdateDataSourceCredentials = (
 export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => {
   const { pipeline_id, node_id } = params
   return useQuery<PipelinePreProcessingParamsResponse>({
-    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
+    queryKey: [NAME_SPACE, 'pipeline-pre-processing-params', pipeline_id, node_id],
     queryFn: () => {
       return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/pre-processing/parameters`, {
         params: {
@@ -270,7 +271,7 @@ export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessin
 export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => {
   const { pipeline_id, node_id } = params
   return useQuery<PipelinePreProcessingParamsResponse>({
-    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
+    queryKey: [NAME_SPACE, 'pipeline-pre-processing-params', pipeline_id, node_id],
     queryFn: () => {
       return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
         params: {
@@ -278,6 +279,7 @@ export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProce
         },
       })
     },
+    staleTime: 0,
     enabled,
   })
 }