Merge branch 'feat/rag-pipeline' into deploy/rag-dev

web/app/components/datasets/documents/create-from-pipeline/process-documents/actions.tsx

@@ -5,11 +5,13 @@ import { RiArrowLeftLine } from '@remixicon/react'
 
 type ActionsProps = {
   onBack: () => void
+  runDisabled?: boolean
   onProcess: () => void
 }
 
 const Actions = ({
   onBack,
+  runDisabled,
   onProcess,
 }: ActionsProps) => {
   const { t } = useTranslation()
@@ -26,6 +28,7 @@ const Actions = ({
       </Button>
       <Button
         variant='primary'
+        disabled={runDisabled}
         onClick={onProcess}
       >
         {t('datasetPipeline.operations.saveAndProcess')}

web/app/components/datasets/documents/create-from-pipeline/process-documents/hooks.ts

@@ -1,22 +1,12 @@
 import { useMemo } from 'react'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline'
-import { PipelineInputVarType } from '@/models/pipeline'
+import { VAR_TYPE_MAP } from '@/models/pipeline'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 
-const VAR_TYPE_MAP: Record<PipelineInputVarType, BaseFieldType> = {
-  [PipelineInputVarType.textInput]: BaseFieldType.textInput,
-  [PipelineInputVarType.paragraph]: BaseFieldType.paragraph,
-  [PipelineInputVarType.select]: BaseFieldType.select,
-  [PipelineInputVarType.singleFile]: BaseFieldType.file,
-  [PipelineInputVarType.multiFiles]: BaseFieldType.fileList,
-  [PipelineInputVarType.number]: BaseFieldType.numberInput,
-  [PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
-}
-
 export const useConfigurations = (datasourceNodeId: string) => {
   const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
-  const { data: paramsConfig } = usePublishedPipelineProcessingParams({
+  const { data: paramsConfig, isFetching: isFetchingParams } = usePublishedPipelineProcessingParams({
     pipeline_id: pipelineId!,
     node_id: datasourceNodeId,
   })
@@ -61,6 +51,7 @@ export const useConfigurations = (datasourceNodeId: string) => {
   }, [paramsConfig])
 
   return {
+    isFetchingParams,
     initialData,
     configurations,
   }

web/app/components/datasets/documents/create-from-pipeline/process-documents/index.tsx

@@ -1,3 +1,4 @@
+import React from 'react'
 import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
 import { useConfigurations } from './hooks'
 import Form from './form'
@@ -20,7 +21,7 @@ const ProcessDocuments = ({
   onBack,
   ref,
 }: ProcessDocumentsProps) => {
-  const { initialData, configurations } = useConfigurations(dataSourceNodeId)
+  const { isFetchingParams, initialData, configurations } = useConfigurations(dataSourceNodeId)
   const schema = generateZodSchema(configurations)
 
   return (
@@ -33,9 +34,9 @@ const ProcessDocuments = ({
         onSubmit={onSubmit}
         onPreview={onPreview}
       />
-      <Actions onBack={onBack} onProcess={onProcess} />
+      <Actions runDisabled={isFetchingParams} onBack={onBack} onProcess={onProcess} />
     </div>
   )
 }
 
-export default ProcessDocuments
+export default React.memo(ProcessDocuments)

web/app/components/datasets/documents/detail/index.tsx

@@ -1,7 +1,7 @@
 'use client'
 import type { FC } from 'react'
 import React, { useMemo, useState } from 'react'
-import { createContext, useContext, useContextSelector } from 'use-context-selector'
+import { createContext, useContextSelector } from 'use-context-selector'
 import { useTranslation } from 'react-i18next'
 import { useRouter } from 'next/navigation'
 import { RiArrowLeftLine, RiLayoutLeft2Line, RiLayoutRight2Line } from '@remixicon/react'
@@ -17,9 +17,9 @@ import style from './style.module.css'
 import cn from '@/utils/classnames'
 import Divider from '@/app/components/base/divider'
 import Loading from '@/app/components/base/loading'
-import { ToastContext } from '@/app/components/base/toast'
+import Toast from '@/app/components/base/toast'
 import type { ChunkingMode, ParentMode, ProcessMode } from '@/models/datasets'
-import { useDatasetDetailContext } from '@/context/dataset-detail'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import FloatRightContainer from '@/app/components/base/float-right-container'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
 import { useCheckSegmentBatchImportProgress, useChildSegmentListKey, useSegmentBatchImport, useSegmentListKey } from '@/service/knowledge/use-segment'
@@ -83,8 +83,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
   const media = useBreakpoints()
   const isMobile = media === MediaType.mobile
 
-  const { notify } = useContext(ToastContext)
+  const dataset = useDatasetDetailContextWithSelector(s => s.dataset)
-  const { dataset } = useDatasetDetailContext()
   const embeddingAvailable = !!dataset?.embedding_available
   const [showMetadata, setShowMetadata] = useState(!isMobile)
   const [newSegmentModalVisible, setNewSegmentModalVisible] = useState(false)
@@ -103,10 +102,10 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
         if (res.job_status === ProcessStatus.WAITING || res.job_status === ProcessStatus.PROCESSING)
           setTimeout(() => checkProcess(res.job_id), 2500)
         if (res.job_status === ProcessStatus.ERROR)
-          notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
+          Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
       },
       onError: (e) => {
-        notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
+        Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
       },
     })
   }
@@ -124,7 +123,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
         checkProcess(res.job_id)
       },
       onError: (e) => {
-        notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
+        Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
       },
     })
   }
@@ -171,12 +170,12 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
   }
 
   const mode = useMemo(() => {
-    return documentDetail?.document_process_rule?.mode
+    return documentDetail?.document_process_rule?.mode || documentDetail?.dataset_process_rule?.mode
-  }, [documentDetail?.document_process_rule])
+  }, [documentDetail?.document_process_rule?.mode, documentDetail?.dataset_process_rule?.mode])
 
   const parentMode = useMemo(() => {
-    return documentDetail?.document_process_rule?.rules?.parent_mode
+    return documentDetail?.document_process_rule?.rules?.parent_mode || documentDetail?.dataset_process_rule?.rules?.parent_mode || 'paragraph'
-  }, [documentDetail?.document_process_rule])
+  }, [documentDetail?.document_process_rule?.rules?.parent_mode, documentDetail?.dataset_process_rule?.rules?.parent_mode])
 
   const isFullDocMode = useMemo(() => {
     return mode === 'hierarchical' && parentMode === 'full-doc'
@@ -260,7 +259,8 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
           {isDetailLoading
             ? <Loading type='app' />
             : <div className={cn('flex h-full min-w-0 grow flex-col',
-              embedding ? '' : isFullDocMode ? 'relative pl-11 pr-11 pt-4' : 'relative pl-5 pr-11 pt-3',
+              !embedding && isFullDocMode && 'relative pl-11 pr-11 pt-4',
+              !embedding && !isFullDocMode && 'relative pl-5 pr-11 pt-3',
             )}>
               {embedding
                 ? <Embedding

web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx

@@ -54,7 +54,7 @@ const Crawler = ({
   const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
 
   const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
-  const { data: paramsConfig } = usePreProcessingParams.current({
+  const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
     pipeline_id: pipelineId!,
     node_id: nodeId,
   }, !!pipelineId && !!nodeId)
@@ -129,6 +129,10 @@ const Crawler = ({
     setCrawlErrorMessage('')
   }, [runDatasourceNode, nodeId, pipelineId, onCheckedCrawlResultChange, checkCrawlStatus, t])
 
+  const handleSubmit = useCallback((value: Record<string, any>) => {
+    handleRun(value)
+  }, [handleRun])
+
   return (
     <div className='flex flex-col'>
       <Header
@@ -139,10 +143,9 @@ const Crawler = ({
         <Options
           variables={paramsConfig?.variables || []}
           isRunning={isRunning}
+          runDisabled={isFetchingParams}
           controlFoldOptions={controlFoldOptions}
-          onSubmit={(value) => {
+          onSubmit={handleSubmit}
-            handleRun(value)
-          }}
         />
       </div>
       {!isInit && (

web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/options/hooks.ts

@@ -1,21 +1,22 @@
-import type { BaseConfiguration, BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
+import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
-import { PipelineInputVarType, type RAGPipelineVariables } from '@/models/pipeline'
+import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
+import { type RAGPipelineVariables, VAR_TYPE_MAP } from '@/models/pipeline'
 import { useMemo } from 'react'
 
 export const useInitialData = (variables: RAGPipelineVariables) => {
   const initialData = useMemo(() => {
-    const initialData: Record<string, any> = {}
+    return variables.reduce((acc, item) => {
-    variables.forEach((item) => {
+      const type = VAR_TYPE_MAP[item.type]
-      if ([PipelineInputVarType.textInput, PipelineInputVarType.paragraph, PipelineInputVarType.select].includes(item.type))
+      if ([BaseFieldType.textInput, BaseFieldType.paragraph, BaseFieldType.select].includes(type))
-        initialData[item.variable] = item.default_value || ''
+        acc[item.variable] = item.default_value ?? ''
-      if (item.type === PipelineInputVarType.number)
+      if (type === BaseFieldType.numberInput)
-        initialData[item.variable] = item.default_value || 0
+        acc[item.variable] = item.default_value ?? 0
-      if ([PipelineInputVarType.singleFile, PipelineInputVarType.multiFiles].includes(item.type))
+      if (type === BaseFieldType.checkbox)
-        initialData[item.variable] = []
+        acc[item.variable] = true
-      if (item.type === PipelineInputVarType.checkbox)
+      if ([BaseFieldType.file, BaseFieldType.fileList].includes(type))
-        initialData[item.variable] = item.default_value || true
+        acc[item.variable] = []
-    })
+      return acc
-    return initialData
+    }, {} as Record<string, any>)
   }, [variables])
 
   return initialData
@@ -26,21 +27,22 @@ export const useConfigurations = (variables: RAGPipelineVariables) => {
     const configurations: BaseConfiguration[] = []
     variables.forEach((item) => {
       configurations.push({
-        type: item.type as unknown as BaseFieldType,
+        type: VAR_TYPE_MAP[item.type],
         variable: item.variable,
         label: item.label,
         required: item.required,
-        placeholder: item.placeholder,
+        maxLength: item.max_length,
-        tooltip: item.tooltips,
         options: item.options?.map(option => ({
           label: option,
           value: option,
         })),
-        maxLength: item.max_length,
         showConditions: [],
-        allowedFileUploadMethods: item.allowed_file_upload_methods,
+        placeholder: item.placeholder,
+        tooltip: item.tooltips,
+        unit: item.unit,
         allowedFileTypes: item.allowed_file_types,
         allowedFileExtensions: item.allowed_file_extensions,
+        allowedFileUploadMethods: item.allowed_file_upload_methods,
       })
     })
     return configurations

web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/options/index.tsx

@@ -17,6 +17,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 type OptionsProps = {
   variables: RAGPipelineVariables
   isRunning: boolean
+  runDisabled?: boolean
   controlFoldOptions?: number
   onSubmit: (data: Record<string, any>) => void
 }
@@ -24,6 +25,7 @@ type OptionsProps = {
 const Options = ({
   variables,
   isRunning,
+  runDisabled,
   controlFoldOptions,
   onSubmit,
 }: OptionsProps) => {
@@ -90,7 +92,7 @@ const Options = ({
         <Button
           variant='primary'
           onClick={form.handleSubmit}
-          disabled={isRunning}
+          disabled={runDisabled || isRunning}
           loading={isRunning}
           className='shrink-0 gap-x-0.5'
           spinnerClassName='!ml-0'

web/app/components/rag-pipeline/components/panel/test-run/document-processing/actions.tsx

@@ -7,11 +7,13 @@ import { WorkflowRunningStatus } from '@/app/components/workflow/types'
 
 type ActionsProps = {
   formParams: CustomActionsProps
+  runDisabled?: boolean
   onBack: () => void
 }
 
 const Actions = ({
   formParams,
+  runDisabled,
   onBack,
 }: ActionsProps) => {
   const { t } = useTranslation()
@@ -32,7 +34,7 @@ const Actions = ({
         onClick={() => {
           form.handleSubmit()
         }}
-        disabled={isSubmitting || !canSubmit || isRunning}
+        disabled={runDisabled || isSubmitting || !canSubmit || isRunning}
         loading={isSubmitting || isRunning}
       >
         {t('datasetPipeline.operations.process')}

web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts

@@ -3,21 +3,11 @@ import type { BaseConfiguration } from '@/app/components/base/form/form-scenario
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useStore } from '@/app/components/workflow/store'
 import { useDraftPipelineProcessingParams } from '@/service/use-pipeline'
-import { PipelineInputVarType } from '@/models/pipeline'
+import { VAR_TYPE_MAP } from '@/models/pipeline'
-
-const VAR_TYPE_MAP: Record<PipelineInputVarType, BaseFieldType> = {
-  [PipelineInputVarType.textInput]: BaseFieldType.textInput,
-  [PipelineInputVarType.paragraph]: BaseFieldType.paragraph,
-  [PipelineInputVarType.select]: BaseFieldType.select,
-  [PipelineInputVarType.singleFile]: BaseFieldType.file,
-  [PipelineInputVarType.multiFiles]: BaseFieldType.fileList,
-  [PipelineInputVarType.number]: BaseFieldType.numberInput,
-  [PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
-}
 
 export const useConfigurations = (datasourceNodeId: string) => {
   const pipelineId = useStore(state => state.pipelineId)
-  const { data: paramsConfig } = useDraftPipelineProcessingParams({
+  const { data: paramsConfig, isFetching: isFetchingParams } = useDraftPipelineProcessingParams({
     pipeline_id: pipelineId!,
     node_id: datasourceNodeId,
   })
@@ -62,6 +52,7 @@ export const useConfigurations = (datasourceNodeId: string) => {
   }, [paramsConfig])
 
   return {
+    isFetchingParams,
     initialData,
     configurations,
   }

web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx

@@ -1,8 +1,8 @@
+import React, { useCallback } from 'react'
 import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
 import { useConfigurations } from './hooks'
 import Options from './options'
 import Actions from './actions'
-import { useCallback } from 'react'
 import type { CustomActionsProps } from '@/app/components/base/form/components/form/actions'
 
 type DocumentProcessingProps = {
@@ -16,12 +16,12 @@ const DocumentProcessing = ({
   onProcess,
   onBack,
 }: DocumentProcessingProps) => {
-  const { initialData, configurations } = useConfigurations(dataSourceNodeId)
+  const { isFetchingParams, initialData, configurations } = useConfigurations(dataSourceNodeId)
   const schema = generateZodSchema(configurations)
 
   const renderCustomActions = useCallback((props: CustomActionsProps) => (
-    <Actions formParams={props} onBack={onBack} />
+    <Actions runDisabled={isFetchingParams} formParams={props} onBack={onBack} />
-  ), [onBack])
+  ), [isFetchingParams, onBack])
 
   return (
     <Options
@@ -34,4 +34,4 @@ const DocumentProcessing = ({
   )
 }
 
-export default DocumentProcessing
+export default React.memo(DocumentProcessing)

web/models/pipeline.ts

@@ -5,6 +5,7 @@ import type { Dependency } from '@/app/components/plugins/types'
 import type { AppIconSelection } from '@/app/components/base/app-icon-picker'
 import type { Viewport } from 'reactflow'
 import type { TransferMethod } from '@/types/app'
+import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 
 export enum DatasourceType {
   localFile = 'local_file',
@@ -119,6 +120,16 @@ export enum PipelineInputVarType {
   checkbox = 'checkbox',
 }
 
+export const VAR_TYPE_MAP: Record<PipelineInputVarType, BaseFieldType> = {
+  [PipelineInputVarType.textInput]: BaseFieldType.textInput,
+  [PipelineInputVarType.paragraph]: BaseFieldType.paragraph,
+  [PipelineInputVarType.select]: BaseFieldType.select,
+  [PipelineInputVarType.singleFile]: BaseFieldType.file,
+  [PipelineInputVarType.multiFiles]: BaseFieldType.fileList,
+  [PipelineInputVarType.number]: BaseFieldType.numberInput,
+  [PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
+}
+
 export type RAGPipelineVariable = {
   belong_to_node_id: string // indicates belong to which node or 'shared'
   type: PipelineInputVarType