refactor: enhance datasource handling by adding fileExtensions support

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -36,6 +36,7 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
         description: node.data.desc || '',
         docTitle: '', // todo: Add docTitle and docLink if needed, or remove these properties if not used
         docLink: '',
+        fileExtensions: node.data.fileExtensions || [],
       }
     })
   }, [datasourceNodes])

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -265,6 +265,7 @@ const TestRunPanel = () => {
                 {datasource?.type === DatasourceType.localFile && (
                   <LocalFile
                     files={fileList}
+                    allowedExtensions={datasource?.fileExtensions || []}
                     updateFile={updateFile}
                     updateFileList={updateFileList}
                     onPreview={updateCurrentFile}

web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx

@@ -14,12 +14,13 @@ import { LanguagesSupported } from '@/i18n/language'
 import { IS_CE_EDITION } from '@/config'
 import { Theme } from '@/types/app'
 import useTheme from '@/hooks/use-theme'
-import { useFileSupportTypes, useFileUploadConfig } from '@/service/use-common'
+import { useFileUploadConfig } from '@/service/use-common'
 
 const FILES_NUMBER_LIMIT = 20
 
 type IFileUploaderProps = {
   fileList: FileItem[]
+  allowedExtensions: string[]
   prepareFileList: (files: FileItem[]) => void
   onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
   onFileListUpdate?: (files: FileItem[]) => void
@@ -29,6 +30,7 @@ type IFileUploaderProps = {
 
 const FileUploader = ({
   fileList,
+  allowedExtensions,
   prepareFileList,
   onFileUpdate,
   onFileListUpdate,
@@ -45,9 +47,7 @@ const FileUploader = ({
   const hideUpload = notSupportBatchUpload && fileList.length > 0
 
   const { data: fileUploadConfigResponse } = useFileUploadConfig()
-  const { data: supportFileTypesResponse } = useFileSupportTypes() // Todo: replace with extensions configured in node
-  const supportTypes = supportFileTypesResponse?.allowed_extensions || []
-  const supportTypesShowNames = (() => {
+  const supportTypesShowNames = useMemo(() => {
     const extensionMap: { [key: string]: string } = {
       md: 'markdown',
       pptx: 'pptx',
@@ -56,14 +56,14 @@ const FileUploader = ({
       docx: 'docx',
     }
 
-    return [...supportTypes]
+    return allowedExtensions
       .map(item => extensionMap[item] || item) // map to standardized extension
       .map(item => item.toLowerCase()) // convert to lower case
       .filter((item, index, self) => self.indexOf(item) === index) // remove duplicates
       .map(item => item.toUpperCase()) // convert to upper case
       .join(locale !== LanguagesSupported[1] ? ', ' : '、 ')
-  })()
-  const ACCEPTS = supportTypes.map((ext: string) => `.${ext}`)
+  }, [locale, allowedExtensions])
+  const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`)
   const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
     file_size_limit: 15,
     batch_count_limit: 5,
@@ -266,7 +266,7 @@ const FileUploader = ({
 
             <span>
               {t('datasetCreation.stepOne.uploader.button')}
-              {supportTypes.length > 0 && (
+              {allowedExtensions.length > 0 && (
                 <label className='ml-1 cursor-pointer text-text-accent' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
               )}
             </span>

web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx

@@ -3,6 +3,7 @@ import FileUploader from './file-uploader'
 
 type LocalFileProps = {
   files: FileItem[]
+  allowedExtensions: string[]
   updateFileList: (files: FileItem[]) => void
   updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
   onPreview?: (file: File) => void
@@ -11,6 +12,7 @@ type LocalFileProps = {
 
 const LocalFile = ({
   files,
+  allowedExtensions,
   updateFileList,
   updateFile,
   onPreview,
@@ -19,6 +21,7 @@ const LocalFile = ({
   return (
     <FileUploader
       fileList={files}
+      allowedExtensions={allowedExtensions}
       prepareFileList={updateFileList}
       onFileListUpdate={updateFileList}
       onFileUpdate={updateFile}

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -31,9 +31,10 @@ export const useDatasourceOptions = () => {
         nodeId: node.id,
         type: node.data.provider_type as DatasourceType,
         variables: node.data.variables || [],
-        description: node.data.desc || '',
+        description: '', // todo: Add description
         docTitle: '', // todo: Add docTitle and docLink
         docLink: '',
+        fileExtensions: node.data.fileExtensions || [],
       }
     })
   }, [datasourceNodes])

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -157,6 +157,7 @@ const TestRunPanel = () => {
                 {datasource?.type === DatasourceType.localFile && (
                   <LocalFile
                     files={fileList}
+                    allowedExtensions={datasource?.fileExtensions || []}
                     updateFile={updateFile}
                     updateFileList={updateFileList}
                     notSupportBatchUpload={false} // only support single file upload in test run

web/app/components/rag-pipeline/components/panel/test-run/types.ts

@@ -19,4 +19,5 @@ export type Datasource = {
   description: string
   docTitle?: string
   docLink?: string
+  fileExtensions?: string[]
 }