refactor: refactor data source handling and add form for document processing

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -51,7 +51,6 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
       return {
         nodeId: node.id,
         type: node.data.provider_type as DatasourceType,
-        variables: node.data.variables || [],
         description: node.data.desc || '',
         docTitle: '', // todo: Add docTitle and docLink if needed, or remove these properties if not used
         docLink: '',

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -274,7 +274,7 @@ const CreateFormPipeline = () => {
                 {datasource?.type === DatasourceType.websiteCrawl && (
                   <WebsiteCrawl
                     nodeId={datasource?.nodeId || ''}
-                    variables={datasource?.variables}
+                    variables={[]} // todo: replace with actual variables if needed
                     headerInfo={{
                       title: datasource.description,
                       docTitle: datasource.docTitle || '',

web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx

@@ -1,7 +1,18 @@
 import React from 'react'
 import { useTranslation } from 'react-i18next'
+import DataSourceOptions from '../../panel/test-run/data-source-options'
+import Form from './form'
+import type { Datasource } from '../../panel/test-run/types'
 
-const DataSource = () => {
+type DatasourceProps = {
+  onSelect: (dataSource: Datasource) => void
+  datasourceNodeId: string
+}
+
+const DataSource = ({
+  onSelect: setDatasource,
+  datasourceNodeId,
+}: DatasourceProps) => {
   const { t } = useTranslation()
 
   return (
@@ -9,6 +20,13 @@ const DataSource = () => {
       <div className='system-sm-semibold-uppercase px-4 pt-2 text-text-secondary'>
         {t('datasetPipeline.inputFieldPanel.preview.stepOneTitle')}
       </div>
+      <div className='px-4 py-2'>
+        <DataSourceOptions
+          onSelect={setDatasource}
+          datasourceNodeId={datasourceNodeId}
+        />
+      </div>
+      <Form variables={[]} />
     </div>
   )
 }

web/app/components/rag-pipeline/components/input-field/preview/form.tsx

@@ -0,0 +1,41 @@
+import { useAppForm } from '@/app/components/base/form'
+import BaseField from '@/app/components/base/form/form-scenarios/base/field'
+import type { RAGPipelineVariables } from '@/models/pipeline'
+import { useConfigurations, useInitialData } from '../../panel/test-run/data-source/website-crawl/base/options/hooks'
+
+type FormProps = {
+  variables: RAGPipelineVariables
+}
+
+const Form = ({
+  variables,
+}: FormProps) => {
+  const initialData = useInitialData(variables)
+  const configurations = useConfigurations(variables)
+
+  const form = useAppForm({
+    defaultValues: initialData,
+  })
+
+  return (
+    <form
+      className='w-full'
+      onSubmit={(e) => {
+        e.preventDefault()
+        e.stopPropagation()
+      }}
+    >
+      <div className='flex flex-col gap-y-3 px-4 py-3'>
+        {configurations.map((config, index) => {
+          const FieldComponent = BaseField({
+            initialData,
+            config,
+          })
+          return <FieldComponent key={index} form={form} />
+        })}
+      </div>
+    </form>
+  )
+}
+
+export default Form

web/app/components/rag-pipeline/components/input-field/preview/index.tsx

@@ -1,7 +1,12 @@
+import { useState } from 'react'
 import { RiCloseLine } from '@remixicon/react'
 import DialogWrapper from './dialog-wrapper'
 import { useTranslation } from 'react-i18next'
 import Badge from '@/app/components/base/badge'
+import DataSource from './data-source'
+import Divider from '@/app/components/base/divider'
+import ProcessDocuments from './process-documents'
+import type { Datasource } from '../../panel/test-run/types'
 
 type PreviewPanelProps = {
   show: boolean
@@ -13,6 +18,7 @@ const PreviewPanel = ({
   onClose,
 }: PreviewPanelProps) => {
   const { t } = useTranslation()
+  const [datasource, setDatasource] = useState<Datasource>()
 
   return (
     <DialogWrapper
@@ -34,6 +40,14 @@ const PreviewPanel = ({
           <RiCloseLine className='size-4 text-text-tertiary' />
         </button>
       </div>
+      <DataSource
+        onSelect={setDatasource}
+        datasourceNodeId={datasource?.nodeId || ''}
+      />
+      <div className='px-4 py-2'>
+        <Divider type='horizontal' className='bg-divider-subtle' />
+      </div>
+      <ProcessDocuments dataSourceNodeId={datasource?.nodeId || ''} />
     </DialogWrapper>
   )
 }

web/app/components/rag-pipeline/components/input-field/preview/process-documents.tsx

@@ -0,0 +1,31 @@
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { useStore } from '@/app/components/workflow/store'
+import { useDraftPipelineProcessingParams } from '@/service/use-pipeline'
+import Form from './form'
+
+type ProcessDocumentsProps = {
+  dataSourceNodeId: string
+}
+
+const ProcessDocuments = ({
+  dataSourceNodeId,
+}: ProcessDocumentsProps) => {
+  const { t } = useTranslation()
+  const pipelineId = useStore(state => state.pipelineId)
+  const { data: paramsConfig } = useDraftPipelineProcessingParams({
+    pipeline_id: pipelineId!,
+    node_id: dataSourceNodeId,
+  })
+
+  return (
+    <div className='flex flex-col'>
+      <div className='system-sm-semibold-uppercase px-4 pt-2 text-text-secondary'>
+        {t('datasetPipeline.inputFieldPanel.preview.stepTwoTitle')}
+      </div>
+      <Form variables={paramsConfig?.variables || []} />
+    </div>
+  )
+}
+
+export default React.memo(ProcessDocuments)

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -49,7 +49,6 @@ export const useDatasourceOptions = () => {
       return {
         nodeId: node.id,
         type: node.data.provider_type as DatasourceType,
-        variables: node.data.variables || [],
         description: '', // todo: Add description
         docTitle: '', // todo: Add docTitle and docLink
         docLink: '',

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -139,7 +139,7 @@ const TestRunPanel = () => {
                 {datasource?.type === DatasourceType.websiteCrawl && (
                   <WebsiteCrawl
                     nodeId={datasource?.nodeId || ''}
-                    variables={datasource?.variables}
+                    variables={[]} // todo: replace with actual variables if needed
                     checkedCrawlResult={websitePages}
                     headerInfo={{
                       title: datasource.description,

web/app/components/rag-pipeline/components/panel/test-run/types.ts

@@ -1,5 +1,5 @@
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
-import type { DatasourceType, RAGPipelineVariables } from '@/models/pipeline'
+import type { DatasourceType } from '@/models/pipeline'
 
 export enum TestRunStep {
   dataSource = 'dataSource',
@@ -15,7 +15,6 @@ export type DataSourceOption = {
 export type Datasource = {
   nodeId: string
   type: DatasourceType
-  variables: RAGPipelineVariables
   description: string
   docTitle?: string
   docLink?: string

web/service/use-pipeline.ts

@@ -148,6 +148,7 @@ export const useDraftPipelineProcessingParams = (params: PipelineProcessingParam
       })
     },
     staleTime: 0,
+    enabled: !!pipeline_id && !!node_id,
   })
 }