refactor: simplify type definitions in form components and update related configurations

web/app/components/base/form/form-scenarios/base/field.tsx

@@ -3,15 +3,15 @@ import { type BaseConfiguration, BaseFieldType } from './types'
 import { withForm } from '../..'
 import { useStore } from '@tanstack/react-form'
 
-type BaseFieldProps<T> = {
-  initialData?: T
-  config: BaseConfiguration<T>
+type BaseFieldProps = {
+  initialData?: Record<string, any>
+  config: BaseConfiguration
 }
 
-const BaseField = <T,>({
+const BaseField = ({
   initialData,
   config,
-}: BaseFieldProps<T>) => withForm({
+}: BaseFieldProps) => withForm({
   defaultValues: initialData,
   render: function Render({
     form,

web/app/components/base/form/form-scenarios/base/index.tsx

@@ -4,14 +4,14 @@ import BaseField from './field'
 import type { BaseFormProps } from './types'
 import { generateZodSchema } from './utils'
 
-const BaseForm = <T,>({
+const BaseForm = ({
   initialData,
   configurations,
   onSubmit,
   CustomActions,
-}: BaseFormProps<T>) => {
+}: BaseFormProps) => {
   const schema = useMemo(() => {
-    const schema = generateZodSchema<T>(configurations)
+    const schema = generateZodSchema(configurations)
     return schema
   }, [configurations])
 
@@ -44,7 +44,7 @@ const BaseForm = <T,>({
     >
       <div className='flex flex-col gap-4 px-4 py-2'>
         {configurations.map((config, index) => {
-          const FieldComponent = BaseField<T>({
+          const FieldComponent = BaseField({
             initialData,
             config,
           })

web/app/components/base/form/form-scenarios/base/types.ts

@@ -1,4 +1,3 @@
-import type { DeepKeys } from '@tanstack/react-form'
 import type { FormType } from '../..'
 import type { Option } from '../../../select/pure'
 
@@ -9,8 +8,8 @@ export enum BaseFieldType {
   select = 'select',
 }
 
-export type ShowCondition<T> = {
-  variable: DeepKeys<T>
+export type ShowCondition = {
+  variable: string
   value: any
 }
 
@@ -29,21 +28,21 @@ export type SelectConfiguration = {
   }
 }
 
-export type BaseConfiguration<T> = {
+export type BaseConfiguration = {
   label: string
-  variable: DeepKeys<T> // Variable name
+  variable: string // Variable name
   maxLength?: number // Max length for text input
   placeholder?: string
   required: boolean
   showOptional?: boolean // show optional label
-  showConditions: ShowCondition<T>[] // Show this field only when all conditions are met
+  showConditions: ShowCondition[] // Show this field only when all conditions are met
   type: BaseFieldType
   tooltip?: string // Tooltip for this field
 } & NumberConfiguration & Partial<SelectConfiguration>
 
-export type BaseFormProps<T> = {
-  initialData?: T
-  configurations: BaseConfiguration<T>[]
+export type BaseFormProps = {
+  initialData?: Record<string, any>
+  configurations: BaseConfiguration[]
   CustomActions?: (form: FormType) => React.ReactNode
-  onSubmit: (value: T) => void
+  onSubmit: (value: Record<string, any>) => void
 }

web/app/components/base/form/form-scenarios/base/utils.ts

@@ -2,7 +2,7 @@ import type { ZodNumber, ZodSchema, ZodString } from 'zod'
 import { z } from 'zod'
 import { type BaseConfiguration, BaseFieldType } from './types'
 
-export const generateZodSchema = <T>(fields: BaseConfiguration<T>[]) => {
+export const generateZodSchema = (fields: BaseConfiguration[]) => {
   const shape: Record<string, ZodSchema> = {}
 
   fields.forEach((field) => {

web/app/components/base/form/form-scenarios/input-field/field.tsx

@@ -3,15 +3,15 @@ import { type InputFieldConfiguration, InputFieldType } from './types'
 import { withForm } from '../..'
 import { useStore } from '@tanstack/react-form'
 
-type InputFieldProps<T> = {
-  initialData?: T
-  config: InputFieldConfiguration<T>
+type InputFieldProps = {
+  initialData?: Record<string, any>
+  config: InputFieldConfiguration
 }
 
-const InputField = <T,>({
+const InputField = ({
   initialData,
   config,
-}: InputFieldProps<T>) => withForm({
+}: InputFieldProps) => withForm({
   defaultValues: initialData,
   render: function Render({
     form,

web/app/components/base/form/form-scenarios/input-field/types.ts

@@ -23,17 +23,17 @@ export type NumberSliderConfiguration = {
   min?: number
 }
 
-export type InputFieldConfiguration<T> = {
+export type InputFieldConfiguration = {
   label: string
-  variable: DeepKeys<T> // Variable name
+  variable: string // Variable name
   maxLength?: number // Max length for text input
   placeholder?: string
   required: boolean
   showOptional?: boolean // show optional label
-  showConditions: ShowCondition<T>[] // Show this field only when all conditions are met
+  showConditions: ShowCondition[] // Show this field only when all conditions are met
   type: InputFieldType
   tooltip?: string // Tooltip for this field
-  listeners?: FieldListeners<T, DeepKeys<T>> // Listener for this field
+  listeners?: FieldListeners<Record<string, any>, DeepKeys<Record<string, any>>> // Listener for this field
 } & NumberConfiguration & Partial<InputTypeSelectConfiguration>
 & Partial<NumberSliderConfiguration>
 & Partial<SelectConfiguration>

web/app/components/base/form/form-scenarios/input-field/utils.ts

@@ -3,7 +3,7 @@ import { z } from 'zod'
 import { type InputFieldConfiguration, InputFieldType } from './types'
 import { SupportedFileTypes, TransferMethod } from '@/app/components/rag-pipeline/components/input-field/editor/form/schema'
 
-export const generateZodSchema = <T>(fields: InputFieldConfiguration<T>[]) => {
+export const generateZodSchema = (fields: InputFieldConfiguration[]) => {
   const shape: Record<string, ZodSchema> = {}
 
   fields.forEach((field) => {

web/app/components/base/form/form-scenarios/node-panel/field.tsx

@@ -3,15 +3,15 @@ import { type InputFieldConfiguration, InputFieldType } from './types'
 import { withForm } from '../..'
 import { useStore } from '@tanstack/react-form'
 
-type InputFieldProps<T> = {
-  initialData?: T
-  config: InputFieldConfiguration<T>
+type InputFieldProps = {
+  initialData?: Record<string, any>
+  config: InputFieldConfiguration
 }
 
-const NodePanelField = <T,>({
+const NodePanelField = ({
   initialData,
   config,
-}: InputFieldProps<T>) => withForm({
+}: InputFieldProps) => withForm({
   defaultValues: initialData,
   render: function Render({
     form,

web/app/components/base/form/form-scenarios/node-panel/types.ts

@@ -24,17 +24,17 @@ export type NumberSliderConfiguration = {
   min?: number
 }
 
-export type InputFieldConfiguration<T> = {
+export type InputFieldConfiguration = {
   label: string
-  variable: DeepKeys<T> // Variable name
+  variable: string // Variable name
   maxLength?: number // Max length for text input
   placeholder?: string
   required: boolean
   showOptional?: boolean // show optional label
-  showConditions: ShowCondition<T>[] // Show this field only when all conditions are met
+  showConditions: ShowCondition[] // Show this field only when all conditions are met
   type: InputFieldType
   tooltip?: string // Tooltip for this field
-  listeners?: FieldListeners<T, DeepKeys<T>> // Listener for this field
+  listeners?: FieldListeners<Record<string, any>, DeepKeys<Record<string, any>>> // Listener for this field
 } & NumberConfiguration & Partial<InputTypeSelectConfiguration>
 & Partial<NumberSliderConfiguration>
 & Partial<SelectConfiguration>

web/app/components/base/form/index.tsx

@@ -10,6 +10,7 @@ import InputTypeSelectField from './components/field/input-type-select'
 import FileTypesField from './components/field/file-types'
 import UploadMethodField from './components/field/upload-method'
 import NumberSliderField from './components/field/number-slider'
+import VariableOrConstantInputField from './components/field/variable-selector'
 
 export const { fieldContext, useFieldContext, formContext, useFormContext }
   = createFormHookContexts()
@@ -26,6 +27,7 @@ export const { useAppForm, withForm } = createFormHook({
     FileTypesField,
     UploadMethodField,
     NumberSliderField,
+    VariableOrConstantInputField,
   },
   formComponents: {
     Actions,

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/options.tsx

@@ -3,7 +3,6 @@ import { useAppForm } from '@/app/components/base/form'
 import BaseField from '@/app/components/base/form/form-scenarios/base/field'
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { ArrowDownRoundFill } from '@/app/components/base/icons/src/vender/solid/general'
-import type { CrawlOptions } from '@/models/datasets'
 import cn from '@/utils/classnames'
 import { RiPlayLargeLine } from '@remixicon/react'
 import { useBoolean } from 'ahooks'
@@ -14,17 +13,13 @@ import type { ZodSchema } from 'zod'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
-export type FormData = {
-  url: string
-} & CrawlOptions
-
 type OptionsProps = {
-  initialData: FormData
-  configurations: BaseConfiguration<FormData>[]
+  initialData: Record<string, any>
+  configurations: BaseConfiguration[]
   isRunning: boolean
   controlFoldOptions?: number
   schema: ZodSchema
-  onSubmit: (data: FormData) => void
+  onSubmit: (data: Record<string, any>) => void
 }
 
 const Options = ({
@@ -104,7 +99,7 @@ const Options = ({
       {!fold && (
         <div className='flex flex-col gap-3 border-t border-divider-subtle px-4 py-3'>
           {configurations.map((config, index) => {
-            const FieldComponent = BaseField<FormData>({
+            const FieldComponent = BaseField({
               initialData,
               config,
             })

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/hooks.ts

@@ -1,7 +1,6 @@
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useTranslation } from 'react-i18next'
-import type { FormData } from '../base/options'
 import { z } from 'zod'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -9,7 +8,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export const useConfigurations = () => {
     const { t } = useTranslation()
-    const configurations: BaseConfiguration<FormData>[] = [
+    const configurations: BaseConfiguration[] = [
       {
         type: BaseFieldType.textInput,
         variable: 'url',

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/hooks.ts

@@ -1,7 +1,6 @@
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useTranslation } from 'react-i18next'
-import type { FormData } from '../base/options'
 import { z } from 'zod'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -9,7 +8,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export const useConfigurations = () => {
     const { t } = useTranslation()
-    const configurations: BaseConfiguration<FormData>[] = [
+    const configurations: BaseConfiguration[] = [
       {
         type: BaseFieldType.textInput,
         variable: 'url',

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/hooks.ts

@@ -1,7 +1,6 @@
 import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useTranslation } from 'react-i18next'
-import type { FormData } from '../base/options'
 import { z } from 'zod'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -9,7 +8,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export const useConfigurations = () => {
     const { t } = useTranslation()
-    const configurations: BaseConfiguration<FormData>[] = [
+    const configurations: BaseConfiguration[] = [
       {
         type: BaseFieldType.textInput,
         variable: 'url',

web/models/pipeline.ts

@@ -1,3 +1,4 @@
+import type { InputVarType } from '@/app/components/workflow/types'
 import type { DSLImportMode, DSLImportStatus } from './app'
 import type { ChunkingMode, IconInfo } from './datasets'
 import type { Dependency } from '@/app/components/plugins/types'
@@ -69,3 +70,17 @@ export type ImportPipelineDSLResponse = {
   error: string
   leaked_dependencies: Dependency[]
 }
+
+export type Variables = {
+  type: InputVarType
+  label: string
+  description: string
+  variable: string
+  max_length: number
+  required: boolean
+  options?: string[]
+}
+
+export type PipelineProcessingParamsResponse = {
+  variables: Variables[]
+}

web/service/use-pipeline.ts

@@ -6,6 +6,7 @@ import type {
   ExportPipelineDSLResponse,
   ImportPipelineDSLRequest,
   ImportPipelineDSLResponse,
+  PipelineProcessingParamsResponse,
   PipelineTemplateByIdResponse,
   PipelineTemplateListParams,
   PipelineTemplateListResponse,
@@ -85,3 +86,12 @@ export const useImportPipelineDSL = (
     ...mutationOptions,
   })
 }
+
+export const usePipelineProcessingParams = (pipelineId: string) => {
+  return useQuery<PipelineProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipelineId],
+    queryFn: () => {
+      return get<PipelineProcessingParamsResponse>(`/rag/pipeline/${pipelineId}/workflows/processing/parameters`)
+    },
+  })
+}