refactor: Refactor data source components

web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx

@@ -5,7 +5,7 @@ import { useContext } from 'use-context-selector'
 import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react'
 import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
 import cn from '@/utils/classnames'
-import type { CustomFile as File, FileItem } from '@/models/datasets'
+import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
 import { ToastContext } from '@/app/components/base/toast'
 import SimplePieChart from '@/app/components/base/simple-pie-chart'
 import { upload } from '@/service/base'
@@ -15,35 +15,34 @@ import { IS_CE_EDITION } from '@/config'
 import { Theme } from '@/types/app'
 import useTheme from '@/hooks/use-theme'
 import { useFileUploadConfig } from '@/service/use-common'
+import { useDataSourceStore } from '../store'
+import produce from 'immer'
 
 const FILES_NUMBER_LIMIT = 20
 
 export type LocalFileProps = {
-  fileList: FileItem[]
   allowedExtensions: string[]
-  prepareFileList: (files: FileItem[]) => void
-  onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
-  onFileListUpdate?: (files: FileItem[]) => void
-  onPreview?: (file: File) => void
   notSupportBatchUpload?: boolean
 }
 
 const LocalFile = ({
-  fileList,
   allowedExtensions,
-  prepareFileList,
-  onFileUpdate,
-  onFileListUpdate,
-  onPreview,
   notSupportBatchUpload,
 }: LocalFileProps) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
   const { locale } = useContext(I18n)
+  const fileList = useDataSourceStore(state => state.localFileList)
+  const setFileList = useDataSourceStore(state => state.setLocalFileList)
+  const setCurrentFile = useDataSourceStore(state => state.setCurrentLocalFile)
+  const previewFileRef = useDataSourceStore(state => state.previewLocalFileRef)
   const [dragging, setDragging] = useState(false)
+
   const dropRef = useRef<HTMLDivElement>(null)
   const dragRef = useRef<HTMLDivElement>(null)
   const fileUploader = useRef<HTMLInputElement>(null)
+  const fileListRef = useRef<FileItem[]>([])
+
   const hideUpload = notSupportBatchUpload && fileList.length > 0
 
   const { data: fileUploadConfigResponse } = useFileUploadConfig()
@@ -69,7 +68,26 @@ const LocalFile = ({
     batch_count_limit: 5,
   }, [fileUploadConfigResponse])
 
-  const fileListRef = useRef<FileItem[]>([])
+  const updateFile = useCallback((fileItem: FileItem, progress: number, list: FileItem[]) => {
+    const newList = produce(list, (draft) => {
+      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
+      draft[targetIndex] = {
+        ...draft[targetIndex],
+        progress,
+      }
+    })
+    setFileList(newList)
+    previewFileRef.current = newList[0].file as DocumentItem
+  }, [previewFileRef, setFileList])
+
+  const updateFileList = useCallback((preparedFiles: FileItem[]) => {
+    setFileList(preparedFiles)
+  }, [setFileList])
+
+  const handlePreview = useCallback((file: File) => {
+    if (file.id)
+      setCurrentFile(file)
+  }, [setCurrentFile])
 
   // utils
   const getFileType = (currentFile: File) => {
@@ -107,7 +125,7 @@ const LocalFile = ({
     const onProgress = (e: ProgressEvent) => {
       if (e.lengthComputable) {
         const percent = Math.floor(e.loaded / e.total * 100)
-        onFileUpdate(fileItem, percent, fileListRef.current)
+        updateFile(fileItem, percent, fileListRef.current)
       }
     }
 
@@ -124,16 +142,16 @@ const LocalFile = ({
         }
         const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
         fileListRef.current[index] = completeFile
-        onFileUpdate(completeFile, 100, fileListRef.current)
+        updateFile(completeFile, 100, fileListRef.current)
         return Promise.resolve({ ...completeFile })
       })
       .catch((e) => {
         notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
-        onFileUpdate(fileItem, -2, fileListRef.current)
+        updateFile(fileItem, -2, fileListRef.current)
         return Promise.resolve({ ...fileItem })
       })
       .finally()
-  }, [fileListRef, notify, onFileUpdate, t])
+  }, [fileListRef, notify, updateFile, t])
 
   const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
     bFiles.forEach(bf => (bf.progress = 0))
@@ -172,10 +190,10 @@ const LocalFile = ({
       progress: -1,
     }))
     const newFiles = [...fileListRef.current, ...preparedFiles]
-    prepareFileList(newFiles)
+    updateFileList(newFiles)
     fileListRef.current = newFiles
     uploadMultipleFiles(preparedFiles)
-  }, [prepareFileList, uploadMultipleFiles, notify, t, fileList])
+  }, [updateFileList, uploadMultipleFiles, notify, t, fileList])
 
   const handleDragEnter = (e: DragEvent) => {
     e.preventDefault()
@@ -207,17 +225,17 @@ const LocalFile = ({
     initialUpload(validFiles)
   }, [initialUpload, isValid, notSupportBatchUpload])
 
-  const selectHandle = () => {
+  const selectHandle = useCallback(() => {
     if (fileUploader.current)
       fileUploader.current.click()
-  }
+  }, [])
 
   const removeFile = (fileID: string) => {
     if (fileUploader.current)
       fileUploader.current.value = ''
 
     fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID)
-    onFileListUpdate?.([...fileListRef.current])
+    updateFileList([...fileListRef.current])
   }
   const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
     const files = [...(e.target.files ?? [])] as File[]
@@ -287,7 +305,7 @@ const LocalFile = ({
             return (
               <div
                 key={`${fileItem.fileID}-${index}`}
-                onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
+                onClick={handlePreview.bind(null, fileItem.file)}
                 className={cn(
                   'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
                   isError && 'border-state-destructive-border bg-state-destructive-hover',

web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx

@@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo } from 'react'
 import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
 import SearchInput from '@/app/components/base/notion-page-selector/search-input'
 import PageSelector from './page-selector'
-import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
+import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
 import Header from '@/app/components/datasets/create/website/base/header'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { DatasourceType } from '@/models/pipeline'
@@ -10,43 +10,44 @@ import { ssePost } from '@/service/base'
 import Toast from '@/app/components/base/toast'
 import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import { useDataSourceStore } from '../store'
 
 type OnlineDocumentsProps = {
-  onSelect: (selectedPages: NotionPage[]) => void
-  previewPageId?: string
-  onPreview?: (selectedPage: NotionPage) => void
   isInPipeline?: boolean
   nodeId: string
   nodeData: DataSourceNodeType
-  documentsData: DataSourceNotionWorkspace[]
-  setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => void
-  searchValue: string
-  setSearchValue: (value: string) => void
-  currentWorkspaceId: string
-  setCurrentWorkspaceId: (workspaceId: string) => void
-  PagesMapAndSelectedPagesId: DataSourceNotionPageMap
-  selectedPagesId: Set<string>
-  setSelectedPagesId: (selectedPagesId: Set<string>) => void
 }
 
 const OnlineDocuments = ({
-  onSelect,
-  previewPageId,
-  onPreview,
   isInPipeline = false,
   nodeId,
   nodeData,
-  documentsData,
-  setDocumentsData,
-  searchValue,
-  setSearchValue,
-  currentWorkspaceId,
-  setCurrentWorkspaceId,
-  PagesMapAndSelectedPagesId,
-  selectedPagesId,
-  setSelectedPagesId,
 }: OnlineDocumentsProps) => {
   const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const documentsData = useDataSourceStore(state => state.documentsData)
+  const setDocumentsData = useDataSourceStore(state => state.setDocumentsData)
+  const searchValue = useDataSourceStore(state => state.searchValue)
+  const setSearchValue = useDataSourceStore(state => state.setSearchValue)
+  const currentWorkspaceId = useDataSourceStore(state => state.currentWorkspaceId)
+  const setCurrentWorkspaceId = useDataSourceStore(state => state.setCurrentWorkspaceId)
+  const setOnlineDocuments = useDataSourceStore(state => state.setOnlineDocuments)
+  const setCurrentDocument = useDataSourceStore(state => state.setCurrentDocument)
+  const selectedPagesId = useDataSourceStore(state => state.selectedPagesId)
+  const setSelectedPagesId = useDataSourceStore(state => state.setSelectedPagesId)
+
+  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
+    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+      next.pages.forEach((page) => {
+        prev[page.page_id] = {
+          ...page,
+          workspace_id: next.workspace_id,
+        }
+      })
+
+      return prev
+    }, {})
+    return pagesMap
+  }, [documentsData])
 
   const datasourceNodeRunURL = !isInPipeline
     ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
@@ -96,13 +97,12 @@ const OnlineDocuments = ({
     const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[pageId])
 
     setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
-    onSelect(selectedPages)
+    setOnlineDocuments(selectedPages)
-  }, [setSelectedPagesId, onSelect, PagesMapAndSelectedPagesId])
+  }, [setSelectedPagesId, setOnlineDocuments, PagesMapAndSelectedPagesId])
 
   const handlePreviewPage = useCallback((previewPageId: string) => {
-    if (onPreview)
+    setCurrentDocument(PagesMapAndSelectedPagesId[previewPageId])
-      onPreview(PagesMapAndSelectedPagesId[previewPageId])
+  }, [PagesMapAndSelectedPagesId, setCurrentDocument])
-  }, [PagesMapAndSelectedPagesId, onPreview])
 
   const headerInfo = useMemo(() => {
     return {
@@ -144,7 +144,6 @@ const OnlineDocuments = ({
             pagesMap={PagesMapAndSelectedPagesId}
             onSelect={handleSelectPages}
             canPreview={!isInPipeline}
-            previewPageId={previewPageId}
             onPreview={handlePreviewPage}
             isMultipleChoice={!isInPipeline}
             currentWorkspaceId={currentWorkspaceId}

web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx

@@ -13,7 +13,6 @@ type PageSelectorProps = {
   list: DataSourceNotionPage[]
   onSelect: (selectedPagesId: Set<string>) => void
   canPreview?: boolean
-  previewPageId?: string
   onPreview?: (selectedPageId: string) => void
   isMultipleChoice?: boolean
   currentWorkspaceId: string
@@ -41,14 +40,13 @@ const PageSelector = ({
   list,
   onSelect,
   canPreview = true,
-  previewPageId,
   onPreview,
   isMultipleChoice = true,
   currentWorkspaceId,
 }: PageSelectorProps) => {
   const { t } = useTranslation()
   const [dataList, setDataList] = useState<NotionPageItem[]>([])
-  const [localPreviewPageId, setLocalPreviewPageId] = useState('')
+  const [currentPreviewPageId, setCurrentPreviewPageId] = useState('')
 
   useEffect(() => {
     setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
@@ -71,7 +69,6 @@ const PageSelector = ({
     }
   })
   const currentDataList = searchValue ? searchDataList : dataList
-  const currentPreviewPageId = previewPageId === undefined ? localPreviewPageId : previewPageId
 
   const listMapWithChildrenAndDescendants = useMemo(() => {
     return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
@@ -149,7 +146,7 @@ const PageSelector = ({
     const current = currentDataList[index]
     const pageId = current.page_id
 
-    setLocalPreviewPageId(pageId)
+    setCurrentPreviewPageId(pageId)
 
     if (onPreview)
       onPreview(pageId)

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx

@@ -1,36 +1,77 @@
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 import Header from './header'
-import { useCallback } from 'react'
+import { useCallback, useEffect } from 'react'
 import FileList from './file-list'
-import type { OnlineDriveFile } from '@/models/pipeline'
+import { DatasourceType } from '@/models/pipeline'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { ssePost } from '@/service/base'
+import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
+import Toast from '@/app/components/base/toast'
+import { useDataSourceStore } from '../store'
 
 type OnlineDriveProps = {
+  nodeId: string
   nodeData: DataSourceNodeType
-  prefix: string[]
+  isInPipeline?: boolean
-  setPrefix: (prefix: string[]) => void
-  keywords: string
-  setKeywords: (keywords: string) => void
-  startAfter: string
-  setStartAfter: (startAfter: string) => void
-  selectedFileList: string[]
-  setSelectedFileList: (selectedFileList: string[]) => void
-  fileList: OnlineDriveFile[]
-  setFileList: (fileList: OnlineDriveFile[]) => void
 }
 
 const OnlineDrive = ({
+  nodeId,
   nodeData,
-  prefix,
+  isInPipeline = false,
-  setPrefix,
-  keywords,
-  setKeywords,
-  startAfter,
-  setStartAfter,
-  selectedFileList,
-  setSelectedFileList,
-  fileList,
-  setFileList,
 }: OnlineDriveProps) => {
+  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const prefix = useDataSourceStore(state => state.prefix)
+  const setPrefix = useDataSourceStore(state => state.setPrefix)
+  const keywords = useDataSourceStore(state => state.keywords)
+  const setKeywords = useDataSourceStore(state => state.setKeywords)
+  const bucket = useDataSourceStore(state => state.bucket)
+  const setBucket = useDataSourceStore(state => state.setBucket)
+  const startAfter = useDataSourceStore(state => state.startAfter)
+  const setStartAfter = useDataSourceStore(state => state.setStartAfter)
+  const selectedFileList = useDataSourceStore(state => state.selectedFileList)
+  const setSelectedFileList = useDataSourceStore(state => state.setSelectedFileList)
+  const fileList = useDataSourceStore(state => state.fileList)
+  const setFileList = useDataSourceStore(state => state.setFileList)
+
+  const datasourceNodeRunURL = !isInPipeline
+    ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
+    : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
+
+  const getOnlineDrive = useCallback(async () => {
+    ssePost(
+      datasourceNodeRunURL,
+      {
+        body: {
+          inputs: {
+            prefix: prefix.join('/'),
+            bucket,
+            start_after: startAfter,
+            max_keys: 30, // Adjust as needed
+          },
+          datasource_type: DatasourceType.onlineDrive,
+        },
+      },
+      {
+        onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
+          console.log('Online Drive documents data:', documentsData)
+        },
+        onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
+          Toast.notify({
+            type: 'error',
+            message: error.error,
+          })
+        },
+      },
+    )
+  }, [bucket, datasourceNodeRunURL, prefix, startAfter])
+
+  useEffect(() => {
+    if (!fileList.length)
+      getOnlineDrive()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
   const updateKeywords = useCallback((keywords: string) => {
     setKeywords(keywords)
   }, [setKeywords])

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts

@@ -1,5 +1,5 @@
 import type { StateCreator } from 'zustand'
-import type { DocumentItem, FileItem } from '@/models/datasets'
+import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
 
 export type LocalFileSliceShape = {
   localFileList: FileItem[]

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts

@@ -2,8 +2,8 @@ import type { StateCreator } from 'zustand'
 import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common'
 
 export type OnlineDocumentSliceShape = {
-  documentData: DataSourceNotionWorkspace[]
+  documentsData: DataSourceNotionWorkspace[]
-  setDocumentData: (documentData: DataSourceNotionWorkspace[]) => void
+  setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void
   searchValue: string
   setSearchValue: (searchValue: string) => void
   currentWorkspaceId: string
@@ -14,13 +14,14 @@ export type OnlineDocumentSliceShape = {
   setCurrentDocument: (document: NotionPage | undefined) => void
   selectedPagesId: Set<string>
   setSelectedPagesId: (selectedPagesId: Set<string>) => void
+  previewOnlineDocumentRef: React.MutableRefObject<NotionPage | undefined>
 }
 
 export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set) => {
   return ({
-    documentData: [],
+    documentsData: [],
-    setDocumentData: (documentData: DataSourceNotionWorkspace[]) => set(() => ({
+    setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({
-      documentData,
+      documentsData,
     })),
     searchValue: '',
     setSearchValue: (searchValue: string) => set(() => ({
@@ -42,5 +43,6 @@ export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> =
     setSelectedPagesId: (selectedPagesId: Set<string>) => set(() => ({
       selectedPagesId,
     })),
+    previewOnlineDocumentRef: { current: undefined },
   })
 }

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts

@@ -12,6 +12,8 @@ export type OnlineDriveSliceShape = {
   setSelectedFileList: (selectedFileList: string[]) => void
   fileList: OnlineDriveFile[]
   setFileList: (fileList: OnlineDriveFile[]) => void
+  bucket: string
+  setBucket: (bucket: string) => void
 }
 
 export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set) => {
@@ -36,5 +38,9 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
     setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
       fileList,
     })),
+    bucket: '',
+    setBucket: (bucket: string) => set(() => ({
+      bucket,
+    })),
   })
 }

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts

@@ -13,6 +13,7 @@ export type WebsiteCrawlSliceShape = {
   setStep: (step: CrawlStep) => void
   previewIndex: number
   setPreviewIndex: (index: number) => void
+  previewWebsitePageRef: React.MutableRefObject<CrawlResultItem | undefined>
 }
 
 export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set) => {
@@ -37,5 +38,6 @@ export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (se
     setPreviewIndex: (index: number) => set(() => ({
       previewIndex: index,
     })),
+    previewWebsitePageRef: { current: undefined },
   })
 }

web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx

@@ -1,7 +1,7 @@
 'use client'
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
+import type { CrawlResultItem } from '@/models/datasets'
 import { CrawlStep } from '@/models/datasets'
 import Header from '@/app/components/datasets/create/website/base/header'
 import Options from './base/options'
@@ -20,41 +20,37 @@ import type {
   DataSourceNodeProcessingResponse,
 } from '@/types/pipeline'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import { useDataSourceStore } from '../store'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 export type WebsiteCrawlProps = {
   nodeId: string
   nodeData: DataSourceNodeType
-  crawlResult: CrawlResult | undefined
-  setCrawlResult: (payload: CrawlResult) => void
-  step: CrawlStep
-  setStep: (step: CrawlStep) => void
-  checkedCrawlResult: CrawlResultItem[]
-  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
-  previewIndex?: number
-  onPreview?: (payload: CrawlResultItem, index: number) => void
   isInPipeline?: boolean
 }
 
 const WebsiteCrawl = ({
   nodeId,
   nodeData,
-  crawlResult,
-  setCrawlResult,
-  step,
-  setStep,
-  checkedCrawlResult,
-  onCheckedCrawlResultChange,
-  previewIndex,
-  onPreview,
   isInPipeline = false,
 }: WebsiteCrawlProps) => {
   const { t } = useTranslation()
   const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
   const [totalNum, setTotalNum] = useState(0)
   const [crawledNum, setCrawledNum] = useState(0)
+  const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
   const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const crawlResult = useDataSourceStore(state => state.crawlResult)
+  const setCrawlResult = useDataSourceStore(state => state.setCrawlResult)
+  const step = useDataSourceStore(state => state.step)
+  const setStep = useDataSourceStore(state => state.setStep)
+  const checkedCrawlResult = useDataSourceStore(state => state.websitePages)
+  const setWebsitePages = useDataSourceStore(state => state.setWebsitePages)
+  const previewWebsitePageRef = useDataSourceStore(state => state.previewWebsitePageRef)
+  const previewIndex = useDataSourceStore(state => state.previewIndex)
+  const setCurrentWebsite = useDataSourceStore(state => state.setCurrentWebsite)
+  const setPreviewIndex = useDataSourceStore(state => state.setPreviewIndex)
 
   const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
   const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
@@ -70,13 +66,21 @@ const WebsiteCrawl = ({
   const isInit = step === CrawlStep.init
   const isCrawlFinished = step === CrawlStep.finished
   const isRunning = step === CrawlStep.running
-  const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
   const showError = isCrawlFinished && crawlErrorMessage
-
   const datasourceNodeRunURL = !isInPipeline
     ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
     : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
 
+  const handleCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
+    setWebsitePages(checkedCrawlResult)
+    previewWebsitePageRef.current = checkedCrawlResult[0]
+  }, [previewWebsitePageRef, setWebsitePages])
+
+  const handlePreview = useCallback((website: CrawlResultItem, index: number) => {
+    setCurrentWebsite(website)
+    setPreviewIndex(index)
+  }, [setCurrentWebsite, setPreviewIndex])
+
   const handleRun = useCallback(async (value: Record<string, any>) => {
     setStep(CrawlStep.running)
     ssePost(
@@ -106,7 +110,7 @@ const WebsiteCrawl = ({
             time_consuming: time_consuming ?? 0,
           }
           setCrawlResult(crawlResultData)
-          onCheckedCrawlResultChange(crawlData || []) // default select the crawl result
+          handleCheckedCrawlResultChange(crawlData || []) // default select the crawl result
           setCrawlErrorMessage('')
           setStep(CrawlStep.finished)
         },
@@ -116,7 +120,7 @@ const WebsiteCrawl = ({
         },
       },
     )
-  }, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t])
+  }, [datasourceNodeRunURL, handleCheckedCrawlResultChange, setCrawlResult, setStep, t])
 
   const handleSubmit = useCallback((value: Record<string, any>) => {
     handleRun(value)
@@ -165,10 +169,10 @@ const WebsiteCrawl = ({
               className='mt-2'
               list={crawlResult?.data || []}
               checkedList={checkedCrawlResult}
-              onSelectedChange={onCheckedCrawlResultChange}
+              onSelectedChange={handleCheckedCrawlResultChange}
               usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
               previewIndex={previewIndex}
-              onPreview={onPreview}
+              onPreview={handlePreview}
               isMultipleChoice={!isInPipeline} // only support single choice in test run
             />
           )}

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -4,11 +4,9 @@ import type { DataSourceOption } from '@/app/components/rag-pipeline/components/
 import { useCallback, useMemo, useRef, useState } from 'react'
 import { BlockEnum, type Node } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
-import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets'
+import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
 import { CrawlStep } from '@/models/datasets'
-import produce from 'immer'
+import { useDataSourceStore } from './data-source/store'
-import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
-import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline'
 
 export const useAddDocumentsSteps = () => {
   const { t } = useTranslation()
@@ -58,26 +56,6 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
         data: node.data,
       })
     })
-    if (process.env.NODE_ENV === 'development') {
-      // todo: delete mock data
-      options.push({
-        label: 'Google Drive',
-        value: '123456',
-        // @ts-expect-error mock data
-        data: {
-          datasource_parameters: {},
-          datasource_configurations: {},
-          type: BlockEnum.DataSource,
-          title: 'Google Drive',
-          plugin_id: 'langgenius/google-drive',
-          provider_type: 'online_drive',
-          provider_name: 'google_drive',
-          datasource_name: 'google-drive',
-          datasource_label: 'Google Drive',
-          selected: false,
-        },
-      })
-    }
     return options
   }, [datasourceNodes])
 
@@ -85,102 +63,41 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
 }
 
 export const useLocalFile = () => {
-  const [fileList, setFileList] = useState<FileItem[]>([])
+  const fileList = useDataSourceStore(state => state.localFileList)
-  const [currentFile, setCurrentFile] = useState<File | undefined>()
+  const previewFileRef = useDataSourceStore(state => state.previewLocalFileRef)
-
+  const currentLocalFile = useDataSourceStore(state => state.currentLocalFile)
-  const previewFile = useRef<DocumentItem>()
+  const setCurrentFile = useDataSourceStore(state => state.setCurrentLocalFile)
 
   const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
 
-  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
+  const hidePreviewLocalFile = useCallback(() => {
-    const newList = produce(list, (draft) => {
-      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
-      draft[targetIndex] = {
-        ...draft[targetIndex],
-        progress,
-      }
-    })
-    setFileList(newList)
-    previewFile.current = newList[0].file as DocumentItem
-  }
-
-  const updateFileList = useCallback((preparedFiles: FileItem[]) => {
-    setFileList(preparedFiles)
-  }, [])
-
-  const updateCurrentFile = useCallback((file: File) => {
-    setCurrentFile(file)
-  }, [])
-
-  const hideFilePreview = useCallback(() => {
     setCurrentFile(undefined)
-  }, [])
+  }, [setCurrentFile])
 
   return {
     fileList,
-    previewFile,
+    previewFileRef,
     allFileLoaded,
-    updateFile,
+    currentLocalFile,
-    updateFileList,
+    hidePreviewLocalFile,
-    currentFile,
-    updateCurrentFile,
-    hideFilePreview,
   }
 }
 
 export const useOnlineDocuments = () => {
-  const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
+  const onlineDocuments = useDataSourceStore(state => state.onlineDocuments)
-  const [searchValue, setSearchValue] = useState('')
+  const previewOnlineDocumentRef = useDataSourceStore(state => state.previewOnlineDocumentRef)
-  const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
+  const setCurrentDocument = useDataSourceStore(state => state.setCurrentDocument)
-  const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
+  const currentDocument = useDataSourceStore(state => state.currentDocument)
-  const [currentDocument, setCurrentDocument] = useState<NotionPage | undefined>()
-
-  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
-    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
-      next.pages.forEach((page) => {
-        prev[page.page_id] = {
-          ...page,
-          workspace_id: next.workspace_id,
-        }
-      })
-
-      return prev
-    }, {})
-    return pagesMap
-  }, [documentsData])
-  const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])]
-  const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
 
-  const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])
+  const hidePreviewOnlineDocument = useCallback(() => {
-
-  const updateOnlineDocuments = (value: NotionPage[]) => {
-    setOnlineDocuments(value)
-  }
-
-  const updateCurrentPage = useCallback((page: NotionPage) => {
-    setCurrentDocument(page)
-  }, [])
-
-  const hideOnlineDocumentPreview = useCallback(() => {
     setCurrentDocument(undefined)
-  }, [])
+  }, [setCurrentDocument])
 
   return {
-    documentsData,
-    setDocumentsData,
-    searchValue,
-    setSearchValue,
-    currentWorkspaceId,
-    setCurrentWorkspaceId,
-    PagesMapAndSelectedPagesId,
-    selectedPagesId,
-    setSelectedPagesId,
     onlineDocuments,
-    previewOnlineDocument,
-    updateOnlineDocuments,
     currentDocument,
-    updateCurrentPage,
+    previewOnlineDocumentRef,
-    hideOnlineDocumentPreview,
+    hidePreviewOnlineDocument,
   }
 }
 
@@ -224,28 +141,5 @@ export const useWebsiteCrawl = () => {
 }
 
 export const useOnlineDrive = () => {
-  const [prefix, setPrefix] = useState<string[]>([])
+  return {}
-  const [keywords, setKeywords] = useState('')
-  const [startAfter, setStartAfter] = useState('')
-  const [selectedFileList, setSelectedFileList] = useState<string[]>([])
-  const [fileList, setFileList] = useState<OnlineDriveFile[]>([
-    {
-      key: 'Bucket_1',
-      size: 1024, // unit bytes
-      type: OnlineDriveFileType.bucket,
-    },
-  ])
-
-  return {
-    prefix,
-    setPrefix,
-    keywords,
-    setKeywords,
-    startAfter,
-    setStartAfter,
-    selectedFileList,
-    setSelectedFileList,
-    fileList,
-    setFileList,
-  }
 }

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -27,7 +27,7 @@ import Processing from './processing'
 import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
-import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
+import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
 import DataSourceProvider from './data-source/store/provider'
 
 const CreateFormPipeline = () => {
@@ -53,56 +53,24 @@ const CreateFormPipeline = () => {
   } = useAddDocumentsSteps()
   const {
     fileList,
-    previewFile,
+    previewFileRef,
     allFileLoaded,
-    updateFile,
+    currentLocalFile,
-    updateFileList,
+    hidePreviewLocalFile,
-    currentFile,
-    updateCurrentFile,
-    hideFilePreview,
   } = useLocalFile()
   const {
-    documentsData,
-    setDocumentsData,
-    searchValue,
-    setSearchValue,
-    currentWorkspaceId,
-    setCurrentWorkspaceId,
-    PagesMapAndSelectedPagesId,
-    selectedPagesId,
-    setSelectedPagesId,
     onlineDocuments,
-    previewOnlineDocument,
-    updateOnlineDocuments,
     currentDocument,
-    updateCurrentPage,
+    previewOnlineDocumentRef,
-    hideOnlineDocumentPreview,
+    hidePreviewOnlineDocument,
   } = useOnlineDocuments()
   const {
     websitePages,
-    crawlResult,
-    setCrawlResult,
-    step,
-    setStep,
     previewWebsitePage,
-    updataCheckedCrawlResultChange,
     currentWebsite,
-    updateCurrentWebsite,
-    previewIndex,
     hideWebsitePreview,
   } = useWebsiteCrawl()
-  const {
+  // const { } = useOnlineDrive()
-    prefix,
-    setPrefix,
-    keywords,
-    setKeywords,
-    startAfter,
-    setStartAfter,
-    selectedFileList,
-    setSelectedFileList,
-    fileList: onlineDriveFileList,
-    setFileList,
-  } = useOnlineDrive()
 
   const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
   const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
@@ -127,7 +95,7 @@ const CreateFormPipeline = () => {
       return
     const datasourceInfoList: Record<string, any>[] = []
     if (datasourceType === DatasourceType.localFile) {
-      const { id, name, type, size, extension, mime_type } = previewFile.current as File
+      const { id, name, type, size, extension, mime_type } = previewFileRef.current as File
       const documentInfo = {
         related_id: id,
         name,
@@ -141,7 +109,7 @@ const CreateFormPipeline = () => {
       datasourceInfoList.push(documentInfo)
     }
     if (datasourceType === DatasourceType.onlineDocument) {
-      const { workspace_id, ...rest } = previewOnlineDocument.current
+      const { workspace_id, ...rest } = previewOnlineDocumentRef.current!
       const documentInfo = {
         workspace_id,
         page: rest,
@@ -162,7 +130,7 @@ const CreateFormPipeline = () => {
         setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
       },
     })
-  }, [datasource, datasourceType, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
+  }, [datasource, datasourceType, pipelineId, previewFileRef, previewOnlineDocumentRef, previewWebsitePage, runPublishedPipeline])
 
   const handleProcess = useCallback(async (data: Record<string, any>) => {
     if (!datasource)
@@ -230,14 +198,14 @@ const CreateFormPipeline = () => {
   }, [handlePreviewChunks, handleProcess])
 
   const handlePreviewFileChange = useCallback((file: DocumentItem) => {
-    previewFile.current = file
+    previewFileRef.current = file
     onClickPreview()
-  }, [onClickPreview, previewFile])
+  }, [onClickPreview, previewFileRef])
 
   const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
-    previewOnlineDocument.current = page
+    previewOnlineDocumentRef.current = page
     onClickPreview()
-  }, [onClickPreview, previewOnlineDocument])
+  }, [onClickPreview, previewOnlineDocumentRef])
 
   const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
     previewWebsitePage.current = website
@@ -272,59 +240,26 @@ const CreateFormPipeline = () => {
                   />
                   {datasourceType === DatasourceType.localFile && (
                     <LocalFile
-                      fileList={fileList}
                       allowedExtensions={datasource!.nodeData.fileExtensions || []}
-                      prepareFileList={updateFileList}
-                      onFileListUpdate={updateFileList}
-                      onFileUpdate={updateFile}
-                      onPreview={updateCurrentFile}
                       notSupportBatchUpload={notSupportBatchUpload}
                     />
                   )}
                   {datasourceType === DatasourceType.onlineDocument && (
                     <OnlineDocuments
-                      documentsData={documentsData}
-                      setDocumentsData={setDocumentsData}
-                      searchValue={searchValue}
-                      setSearchValue={setSearchValue}
-                      currentWorkspaceId={currentWorkspaceId}
-                      setCurrentWorkspaceId={setCurrentWorkspaceId}
-                      PagesMapAndSelectedPagesId={PagesMapAndSelectedPagesId}
-                      selectedPagesId={selectedPagesId}
-                      setSelectedPagesId={setSelectedPagesId}
                       nodeId={datasource!.nodeId}
                       nodeData={datasource!.nodeData}
-                      onSelect={updateOnlineDocuments}
-                      onPreview={updateCurrentPage}
                     />
                   )}
                   {datasourceType === DatasourceType.websiteCrawl && (
                     <WebsiteCrawl
                       nodeId={datasource!.nodeId}
                       nodeData={datasource!.nodeData}
-                      crawlResult={crawlResult}
-                      setCrawlResult={setCrawlResult}
-                      step={step}
-                      setStep={setStep}
-                      checkedCrawlResult={websitePages}
-                      onCheckedCrawlResultChange={updataCheckedCrawlResultChange}
-                      onPreview={updateCurrentWebsite}
-                      previewIndex={previewIndex}
                     />
                   )}
                   {datasourceType === DatasourceType.onlineDrive && (
                     <OnlineDrive
+                      nodeId={datasource!.nodeId}
                       nodeData={datasource!.nodeData}
-                      prefix={prefix}
-                      setPrefix={setPrefix}
-                      keywords={keywords}
-                      setKeywords={setKeywords}
-                      startAfter={startAfter}
-                      setStartAfter={setStartAfter}
-                      selectedFileList={selectedFileList}
-                      setSelectedFileList={setSelectedFileList}
-                      fileList={onlineDriveFileList}
-                      setFileList={setFileList}
                     />
                   )}
                   {isShowVectorSpaceFull && (
@@ -362,15 +297,25 @@ const CreateFormPipeline = () => {
         currentStep === 1 && (
           <div className='h-full min-w-0 flex-1'>
             <div className='flex h-full flex-col pl-2 pt-2'>
-              {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
+              {currentLocalFile && (
+                <FilePreview
+                  file={currentLocalFile}
+                  hidePreview={hidePreviewLocalFile}
+                />
+              )}
               {currentDocument && (
                 <OnlineDocumentPreview
                   datasourceNodeId={datasource!.nodeId}
                   currentPage={currentDocument}
-                  hidePreview={hideOnlineDocumentPreview}
+                  hidePreview={hidePreviewOnlineDocument}
+                />
+              )}
+              {currentWebsite && (
+                <WebsitePreview
+                  currentWebsite={currentWebsite}
+                  hidePreview={hideWebsitePreview}
                 />
               )}
-              {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
             </div>
           </div>
         )

web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx

@@ -1,12 +1,12 @@
 'use client'
-import React from 'react'
+import React, { useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import Loading from './loading'
 import type { CustomFile as File } from '@/models/datasets'
 import { RiCloseLine } from '@remixicon/react'
 import { useFilePreview } from '@/service/use-common'
 import DocumentFileIcon from '../../../common/document-file-icon'
-import { formatNumberAbbreviated } from '@/utils/format'
+import { formatFileSize, formatNumberAbbreviated } from '@/utils/format'
 
 type FilePreviewProps = {
   file: File
@@ -20,31 +20,19 @@ const FilePreview = ({
   const { t } = useTranslation()
   const { data: fileData, isFetching } = useFilePreview(file.id || '')
 
-  const getFileName = (currentFile?: File) => {
+  const fileName = useMemo(() => {
-    if (!currentFile)
+    if (!file)
       return ''
-    const arr = currentFile.name.split('.')
+    const arr = file.name.split('.')
     return arr.slice(0, -1).join()
-  }
+  }, [file])
-
-  const getFileSize = (fileSize: number) => {
-    if (!fileSize)
-      return fileSize
-    const units = ['', 'K', 'M', 'G', 'T', 'P']
-    let index = 0
-    while (fileSize >= 1024 && index < units.length) {
-      fileSize = fileSize / 1024
-      index++
-    }
-    return `${fileSize.toFixed(2)} ${units[index]}B`
-  }
 
   return (
     <div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
       <div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
         <div className='flex grow flex-col gap-y-1'>
           <div className='system-2xs-semibold-uppercase text-text-accent'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
-          <div className='title-md-semi-bold text-tex-primary'>{`${getFileName(file)}.${file.extension}`}</div>
+          <div className='title-md-semi-bold text-tex-primary'>{`${fileName}.${file.extension || ''}`}</div>
           <div className='system-xs-medium flex items-center gap-x-1 text-text-tertiary'>
             <DocumentFileIcon
               className='size-3.5 shrink-0'
@@ -53,7 +41,7 @@ const FilePreview = ({
             />
             <span className='uppercase'>{file.extension}</span>
             <span>·</span>
-            <span>{getFileSize(file.size)}</span>
+            <span>{formatFileSize(file.size)}</span>
             {fileData && (
               <>
                 <span>·</span>

web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx

@@ -6,12 +6,12 @@ import { RiCloseLine, RiGlobalLine } from '@remixicon/react'
 import { formatNumberAbbreviated } from '@/utils/format'
 
 type WebsitePreviewProps = {
-  payload: CrawlResultItem
+  currentWebsite: CrawlResultItem
   hidePreview: () => void
 }
 
 const WebsitePreview = ({
-  payload,
+  currentWebsite,
   hidePreview,
 }: WebsitePreviewProps) => {
   const { t } = useTranslation()
@@ -21,13 +21,13 @@ const WebsitePreview = ({
       <div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
         <div className='flex grow flex-col gap-y-1'>
           <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
-          <div className='title-md-semi-bold text-tex-primary'>{payload.title}</div>
+          <div className='title-md-semi-bold text-tex-primary'>{currentWebsite.title}</div>
           <div className='system-xs-medium flex gap-x-1  text-text-tertiary'>
             <RiGlobalLine className='size-3.5' />
-            <span className='uppercase' title={payload.source_url}>{payload.source_url}</span>
+            <span className='uppercase' title={currentWebsite.source_url}>{currentWebsite.source_url}</span>
             <span>·</span>
             <span>·</span>
-            <span>{`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
+            <span>{`${formatNumberAbbreviated(currentWebsite.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
           </div>
         </div>
         <button
@@ -39,7 +39,7 @@ const WebsitePreview = ({
         </button>
       </div>
       <div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
-        {payload.markdown}
+        {currentWebsite.markdown}
       </div>
     </div>
   )

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -5,12 +5,7 @@ import { useNodes } from 'reactflow'
 import { BlockEnum } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 import { useCallback, useMemo, useState } from 'react'
-import type { CrawlResult } from '@/models/datasets'
-import { type CrawlResultItem, CrawlStep, type FileItem } from '@/models/datasets'
-import produce from 'immer'
-import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
 import type { OnlineDriveFile } from '@/models/pipeline'
-import { OnlineDriveFileType } from '@/models/pipeline'
 
 export const useTestRunSteps = () => {
   const { t } = useTranslation()
@@ -57,131 +52,27 @@ export const useDatasourceOptions = () => {
         data: node.data,
       })
     })
-    if (process.env.NODE_ENV === 'development') {
-      // todo: delete mock data
-      options.push({
-        label: 'Google Drive',
-        value: '123456',
-        // @ts-expect-error mock data
-        data: {
-          datasource_parameters: {},
-          datasource_configurations: {},
-          type: BlockEnum.DataSource,
-          title: 'Google Drive',
-          plugin_id: 'langgenius/google-drive',
-          provider_type: 'online_drive',
-          provider_name: 'google_drive',
-          datasource_name: 'google-drive',
-          datasource_label: 'Google Drive',
-          selected: false,
-        },
-      })
-    }
     return options
   }, [datasourceNodes])
 
   return options
 }
 
-export const useLocalFile = () => {
-  const [fileList, setFileList] = useState<FileItem[]>([])
-
-  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
-    const newList = produce(list, (draft) => {
-      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
-      draft[targetIndex] = {
-        ...draft[targetIndex],
-        progress,
-      }
-    })
-    setFileList(newList)
-  }
-
-  const updateFileList = (preparedFiles: FileItem[]) => {
-    setFileList(preparedFiles)
-  }
-
-  return {
-    fileList,
-    updateFile,
-    updateFileList,
-  }
-}
-
-export const useOnlineDocuments = () => {
-  const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
-  const [searchValue, setSearchValue] = useState('')
-  const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
-  const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
-
-  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
-    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
-      next.pages.forEach((page) => {
-        prev[page.page_id] = {
-          ...page,
-          workspace_id: next.workspace_id,
-        }
-      })
-
-      return prev
-    }, {})
-    return pagesMap
-  }, [documentsData])
-  const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])]
-  const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
-
-  const updateOnlineDocuments = (value: NotionPage[]) => {
-    setOnlineDocuments(value)
-  }
-
-  return {
-    documentsData,
-    setDocumentsData,
-    searchValue,
-    setSearchValue,
-    currentWorkspaceId,
-    setCurrentWorkspaceId,
-    PagesMapAndSelectedPagesId,
-    selectedPagesId,
-    setSelectedPagesId,
-    onlineDocuments,
-    updateOnlineDocuments,
-  }
-}
-
-export const useWebsiteCrawl = () => {
-  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
-  const [crawlResult, setCrawlResult] = useState<CrawlResult | undefined>()
-  const [step, setStep] = useState<CrawlStep>(CrawlStep.init)
-
-  return {
-    crawlResult,
-    setCrawlResult,
-    websitePages,
-    setWebsitePages,
-    step,
-    setStep,
-  }
-}
-
 export const useOnlineDrive = () => {
   const [prefix, setPrefix] = useState<string[]>([])
   const [keywords, setKeywords] = useState('')
+  const [bucket, setBucket] = useState('')
   const [startAfter, setStartAfter] = useState('')
   const [selectedFileList, setSelectedFileList] = useState<string[]>([])
-  const [fileList, setFileList] = useState<OnlineDriveFile[]>([
+  const [fileList, setFileList] = useState<OnlineDriveFile[]>([])
-    {
-      key: 'Bucket_1',
-      size: 1024, // unit bytes
-      type: OnlineDriveFileType.bucket,
-    },
-  ])
 
   return {
     prefix,
     setPrefix,
     keywords,
     setKeywords,
+    bucket,
+    setBucket,
     startAfter,
     setStartAfter,
     selectedFileList,

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -1,6 +1,6 @@
 import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store'
 import { useCallback, useMemo, useState } from 'react'
-import { useLocalFile, useOnlineDocuments, useOnlineDrive, useTestRunSteps, useWebsiteCrawl } from './hooks'
+import { useTestRunSteps } from './hooks'
 import DataSourceOptions from './data-source-options'
 import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
 import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
@@ -16,9 +16,14 @@ import CloseButton from './close-button'
 import Header from './header'
 import FooterTips from './footer-tips'
 import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider'
+import { useDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
 
 const TestRunPanel = () => {
   const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
+  const fileList = useDataSourceStore(state => state.localFileList)
+  const onlineDocuments = useDataSourceStore(state => state.onlineDocuments)
+  const websitePages = useDataSourceStore(state => state.websitePages)
+  const selectedFileList = useDataSourceStore(state => state.selectedFileList)
   const [datasource, setDatasource] = useState<Datasource>()
 
   const {
@@ -27,45 +32,6 @@ const TestRunPanel = () => {
     handleNextStep,
     handleBackStep,
   } = useTestRunSteps()
-  const {
-    fileList,
-    updateFile,
-    updateFileList,
-  } = useLocalFile()
-  const {
-    documentsData,
-    setDocumentsData,
-    searchValue,
-    setSearchValue,
-    currentWorkspaceId,
-    setCurrentWorkspaceId,
-    PagesMapAndSelectedPagesId,
-    selectedPagesId,
-    setSelectedPagesId,
-    onlineDocuments,
-    updateOnlineDocuments,
-  } = useOnlineDocuments()
-  const {
-    crawlResult,
-    setCrawlResult,
-    websitePages,
-    setWebsitePages,
-    step,
-    setStep,
-  } = useWebsiteCrawl()
-  const {
-    prefix,
-    setPrefix,
-    keywords,
-    setKeywords,
-    startAfter,
-    setStartAfter,
-    selectedFileList,
-    setSelectedFileList,
-    fileList: onlineDriveFileList,
-    setFileList,
-  } = useOnlineDrive()
-  const { handleRun } = useWorkflowRun()
 
   const datasourceType = datasource?.nodeData.provider_type
 
@@ -77,12 +43,16 @@ const TestRunPanel = () => {
       return !onlineDocuments.length
     if (datasourceType === DatasourceType.websiteCrawl)
       return !websitePages.length
+    if (datasourceType === DatasourceType.onlineDrive)
+      return !selectedFileList.length
     return false
-  }, [datasource, datasourceType, fileList, onlineDocuments.length, websitePages.length])
+  }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileList.length, websitePages.length])
 
-  const handleClose = () => {
+  const handleClose = useCallback(() => {
     setShowDebugAndPreviewPanel(false)
-  }
+  }, [setShowDebugAndPreviewPanel])
+
+  const { handleRun } = useWorkflowRun()
 
   const handleProcess = useCallback((data: Record<string, any>) => {
     if (!datasource)
@@ -137,57 +107,29 @@ const TestRunPanel = () => {
                 />
                 {datasourceType === DatasourceType.localFile && (
                   <LocalFile
-                    fileList={fileList}
                     allowedExtensions={datasource!.nodeData.fileExtensions || []}
-                    prepareFileList={updateFileList}
-                    onFileListUpdate={updateFileList}
-                    onFileUpdate={updateFile}
                     notSupportBatchUpload={false} // only support single file upload in test run
                   />
                 )}
                 {datasourceType === DatasourceType.onlineDocument && (
                   <OnlineDocuments
-                    documentsData={documentsData}
-                    setDocumentsData={setDocumentsData}
-                    searchValue={searchValue}
-                    setSearchValue={setSearchValue}
-                    currentWorkspaceId={currentWorkspaceId}
-                    setCurrentWorkspaceId={setCurrentWorkspaceId}
-                    PagesMapAndSelectedPagesId={PagesMapAndSelectedPagesId}
-                    selectedPagesId={selectedPagesId}
-                    setSelectedPagesId={setSelectedPagesId}
                     nodeId={datasource!.nodeId}
                     nodeData={datasource!.nodeData}
-                    onSelect={updateOnlineDocuments}
                     isInPipeline
                   />
                 )}
                 {datasourceType === DatasourceType.websiteCrawl && (
                   <WebsiteCrawl
                     nodeId={datasource!.nodeId}
-                    checkedCrawlResult={websitePages}
                     nodeData={datasource!.nodeData}
-                    crawlResult={crawlResult}
-                    setCrawlResult={setCrawlResult}
-                    step={step}
-                    setStep={setStep}
-                    onCheckedCrawlResultChange={setWebsitePages}
                     isInPipeline
                   />
                 )}
                 {datasourceType === DatasourceType.onlineDrive && (
                   <OnlineDrive
+                    nodeId={datasource!.nodeId}
                     nodeData={datasource!.nodeData}
-                    prefix={prefix}
+                    isInPipeline
-                    setPrefix={setPrefix}
-                    keywords={keywords}
-                    setKeywords={setKeywords}
-                    startAfter={startAfter}
-                    setStartAfter={setStartAfter}
-                    selectedFileList={selectedFileList}
-                    setSelectedFileList={setSelectedFileList}
-                    fileList={onlineDriveFileList}
-                    setFileList={setFileList}
                   />
                 )}
               </div>