refactor: refactor online documents handling and update related components

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -119,31 +119,31 @@ export const useLocalFile = () => {
   }
 }
 
-export const useNotionsPages = () => {
+export const useOnlineDocuments = () => {
-  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
+  const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
-  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
+  const [currentDocuments, setCurrentDocuments] = useState<NotionPage | undefined>()
 
-  const previewNotionPage = useRef<NotionPage>(notionPages[0])
+  const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])
 
-  const updateNotionPages = (value: NotionPage[]) => {
+  const updateOnlineDocuments = (value: NotionPage[]) => {
-    setNotionPages(value)
+    setOnlineDocuments(value)
   }
 
   const updateCurrentPage = useCallback((page: NotionPage) => {
-    setCurrentNotionPage(page)
+    setCurrentDocuments(page)
   }, [])
 
-  const hideNotionPagePreview = useCallback(() => {
+  const hideOnlineDocumentPreview = useCallback(() => {
-    setCurrentNotionPage(undefined)
+    setCurrentDocuments(undefined)
   }, [])
 
   return {
-    notionPages,
+    onlineDocuments,
-    previewNotionPage,
+    previewOnlineDocument,
-    updateNotionPages,
+    updateOnlineDocuments,
-    currentNotionPage,
+    currentDocuments,
     updateCurrentPage,
-    hideNotionPagePreview,
+    hideOnlineDocumentPreview,
   }
 }
 

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -5,7 +5,7 @@ import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEst
 import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
 import { useProviderContextSelector } from '@/context/provider-context'
 import type { NotionPage } from '@/models/common'
-import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
+import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl'
 import Actions from './data-source/actions'
@@ -26,7 +26,7 @@ import Processing from './processing'
 import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
-import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks'
+import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
 
 const CreateFormPipeline = () => {
   const { t } = useTranslation()
@@ -63,13 +63,13 @@ const CreateFormPipeline = () => {
     hideFilePreview,
   } = useLocalFile()
   const {
-    notionPages,
+    onlineDocuments,
-    previewNotionPage,
+    previewOnlineDocument,
-    updateNotionPages,
+    updateOnlineDocuments,
-    currentNotionPage,
+    currentDocuments,
     updateCurrentPage,
-    hideNotionPagePreview,
+    hideOnlineDocumentPreview,
-  } = useNotionsPages()
+  } = useOnlineDocuments()
   const {
     websitePages,
     websiteCrawlJobId,
@@ -90,11 +90,11 @@ const CreateFormPipeline = () => {
     if (datasource.type === DatasourceType.localFile)
       return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
     if (datasource.type === DatasourceType.onlineDocument)
-      return isShowVectorSpaceFull || !notionPages.length
+      return isShowVectorSpaceFull || !onlineDocuments.length
     if (datasource.type === DatasourceType.websiteCrawl)
       return isShowVectorSpaceFull || !websitePages.length
     return false
-  }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
+  }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
 
   const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
 
@@ -117,7 +117,7 @@ const CreateFormPipeline = () => {
       datasourceInfoList.push(documentInfo)
     }
     if (datasource.type === DatasourceType.onlineDocument) {
-      const { workspace_id, ...rest } = previewNotionPage.current
+      const { workspace_id, ...rest } = previewOnlineDocument.current
       const documentInfo = {
         workspace_id,
         page: rest,
@@ -143,7 +143,7 @@ const CreateFormPipeline = () => {
         setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
       },
     })
-  }, [datasource, pipelineId, previewFile, previewNotionPage, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
+  }, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
 
   const handleProcess = useCallback(async (data: Record<string, any>) => {
     if (!datasource)
@@ -166,7 +166,7 @@ const CreateFormPipeline = () => {
       })
     }
     if (datasource.type === DatasourceType.onlineDocument) {
-      notionPages.forEach((page) => {
+      onlineDocuments.forEach((page) => {
         const { workspace_id, ...rest } = page
         const documentInfo = {
           workspace_id,
@@ -196,7 +196,7 @@ const CreateFormPipeline = () => {
         handleNextStep()
       },
     })
-  }, [datasource, fileList, handleNextStep, notionPages, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
+  }, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
 
   const onClickProcess = useCallback(() => {
     isPreview.current = false
@@ -217,10 +217,10 @@ const CreateFormPipeline = () => {
     onClickPreview()
   }, [onClickPreview, previewFile])
 
-  const handlePreviewNotionPageChange = useCallback((page: NotionPage) => {
+  const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
-    previewNotionPage.current = page
+    previewOnlineDocument.current = page
     onClickPreview()
-  }, [onClickPreview, previewNotionPage])
+  }, [onClickPreview, previewOnlineDocument])
 
   const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
     previewWebsitePage.current = website
@@ -263,15 +263,15 @@ const CreateFormPipeline = () => {
                   />
                 )}
                 {datasource?.type === DatasourceType.onlineDocument && (
-                  <Notion
+                  <OnlineDocuments
                     nodeId={datasource?.nodeId || ''}
                     headerInfo={{
                       title: datasource.description,
                       docTitle: datasource.docTitle || '',
                       docLink: datasource.docLink || '',
                     }}
-                    notionPages={notionPages}
+                    onlineDocuments={onlineDocuments}
-                    updateNotionPages={updateNotionPages}
+                    updateOnlineDocuments={updateOnlineDocuments}
                     canPreview
                     onPreview={updateCurrentPage}
                   />
@@ -327,7 +327,7 @@ const CreateFormPipeline = () => {
         currentStep === 1 && (
           <div className='flex h-full w-[752px] shrink-0 pl-2 pt-2'>
             {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
-            {currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
+            {currentDocuments && <NotionPagePreview currentPage={currentDocuments} hidePreview={hideOnlineDocumentPreview} />}
             {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
           </div>
         )
@@ -339,14 +339,14 @@ const CreateFormPipeline = () => {
               <ChunkPreview
                 datasource={datasource!}
                 files={fileList.map(file => file.file)}
-                notionPages={notionPages}
+                onlineDocuments={onlineDocuments}
                 websitePages={websitePages}
                 isIdle={isIdle && isPreview.current}
                 isPending={isPending && isPreview.current}
                 estimateData={estimateData}
                 onPreview={onClickPreview}
                 handlePreviewFileChange={handlePreviewFileChange}
-                handlePreviewNotionPageChange={handlePreviewNotionPageChange}
+                handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
                 handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
               />
             )}

web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx

@@ -20,35 +20,35 @@ import { DatasourceType } from '@/models/pipeline'
 type ChunkPreviewProps = {
   datasource: Datasource
   files: CustomFile[]
-  notionPages: NotionPage[]
+  onlineDocuments: NotionPage[]
   websitePages: CrawlResultItem[]
   isIdle: boolean
   isPending: boolean
   estimateData: FileIndexingEstimateResponse | undefined
   onPreview: () => void
   handlePreviewFileChange: (file: DocumentItem) => void
-  handlePreviewNotionPageChange: (page: NotionPage) => void
+  handlePreviewOnlineDocumentChange: (page: NotionPage) => void
   handlePreviewWebsitePageChange: (page: CrawlResultItem) => void
 }
 
 const ChunkPreview = ({
   datasource,
   files,
-  notionPages,
+  onlineDocuments,
   websitePages,
   isIdle,
   isPending,
   estimateData,
   onPreview,
   handlePreviewFileChange,
-  handlePreviewNotionPageChange,
+  handlePreviewOnlineDocumentChange,
   handlePreviewWebsitePageChange,
 }: ChunkPreviewProps) => {
   const { t } = useTranslation()
   const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form)
 
   const [previewFile, setPreviewFile] = useState<DocumentItem>(files[0] as DocumentItem)
-  const [previewNotionPage, setPreviewNotionPage] = useState<NotionPage>(notionPages[0])
+  const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0])
   const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0])
 
   const dataSourceType = datasource?.type
@@ -72,20 +72,20 @@ const ChunkPreview = ({
           {dataSourceType === DatasourceType.onlineDocument
             && <PreviewDocumentPicker
               files={
-                notionPages.map(page => ({
+                onlineDocuments.map(page => ({
                   id: page.page_id,
                   name: page.page_name,
                   extension: 'md',
                 }))
               }
               onChange={(selected) => {
-                const selectedPage = notionPages.find(page => page.page_id === selected.id)
+                const selectedPage = onlineDocuments.find(page => page.page_id === selected.id)
-                setPreviewNotionPage(selectedPage!)
+                setPreviewOnlineDocument(selectedPage!)
-                handlePreviewNotionPageChange(selectedPage!)
+                handlePreviewOnlineDocumentChange(selectedPage!)
               }}
               value={{
-                id: previewNotionPage?.page_id || '',
+                id: previewOnlineDocument?.page_id || '',
-                name: previewNotionPage?.page_name || '',
+                name: previewOnlineDocument?.page_name || '',
                 extension: 'md',
               }}
             />

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx

@@ -1,35 +1,35 @@
 import type { NotionPage } from '@/models/common'
-import NotionPageSelector from './notion-page-selector'
+import OnlineDocumentSelector from './online-document-selector'
 
-type NotionProps = {
+type OnlineDocumentsProps = {
   nodeId: string
   headerInfo: {
     title: string
     docTitle: string
     docLink: string
   }
-  notionPages: NotionPage[]
+  onlineDocuments: NotionPage[]
-  updateNotionPages: (value: NotionPage[]) => void
+  updateOnlineDocuments: (value: NotionPage[]) => void
   canPreview?: boolean
   onPreview?: (selectedPage: NotionPage) => void
   isInPipeline?: boolean
 }
 
-const Notion = ({
+const OnlineDocuments = ({
   nodeId,
   headerInfo,
-  notionPages,
+  onlineDocuments,
-  updateNotionPages,
+  updateOnlineDocuments,
   canPreview = false,
   onPreview,
   isInPipeline = false,
-}: NotionProps) => {
+}: OnlineDocumentsProps) => {
   return (
-    <NotionPageSelector
+    <OnlineDocumentSelector
       nodeId={nodeId}
       headerInfo={headerInfo}
-      value={notionPages.map(page => page.page_id)}
+      value={onlineDocuments.map(page => page.page_id)}
-      onSelect={updateNotionPages}
+      onSelect={updateOnlineDocuments}
       canPreview={canPreview}
       onPreview={onPreview}
       isInPipeline={isInPipeline}
@@ -37,4 +37,4 @@ const Notion = ({
   )
 }
 
-export default Notion
+export default OnlineDocuments

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx

@@ -8,7 +8,7 @@ import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { useDraftDatasourceNodeRun, usePublishedDatasourceNodeRun } from '@/service/use-pipeline'
 import { DatasourceType } from '@/models/pipeline'
 
-type NotionPageSelectorProps = {
+type OnlineDocumentSelectorProps = {
   value?: string[]
   onSelect: (selectedPages: NotionPage[]) => void
   canPreview?: boolean
@@ -23,7 +23,7 @@ type NotionPageSelectorProps = {
   }
 }
 
-const NotionPageSelector = ({
+const OnlineDocumentSelector = ({
   value,
   onSelect,
   canPreview,
@@ -32,42 +32,42 @@ const NotionPageSelector = ({
   isInPipeline = false,
   nodeId,
   headerInfo,
-}: NotionPageSelectorProps) => {
+}: OnlineDocumentSelectorProps) => {
   const pipeline_id = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
-  const [notionData, setNotionData] = useState<DataSourceNotionWorkspace[]>([])
+  const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
   const [searchValue, setSearchValue] = useState('')
   const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
 
   const useDatasourceNodeRun = useRef(!isInPipeline ? usePublishedDatasourceNodeRun : useDraftDatasourceNodeRun)
-  const { mutateAsync: getNotionPages } = useDatasourceNodeRun.current()
+  const { mutateAsync: crawlOnlineDocuments } = useDatasourceNodeRun.current()
 
-  const getNotionData = useCallback(async () => {
+  const getOnlineDocuments = useCallback(async () => {
     if (pipeline_id) {
-      await getNotionPages({
+      await crawlOnlineDocuments({
         pipeline_id,
         node_id: nodeId,
         inputs: {},
         datasource_type: DatasourceType.onlineDocument,
       }, {
-        onSuccess(notionData) {
+        onSuccess(documentsData) {
-          setNotionData(notionData as DataSourceNotionWorkspace[])
+          setDocumentsData(documentsData as DataSourceNotionWorkspace[])
         },
       })
     }
-  }, [getNotionPages, nodeId, pipeline_id])
+  }, [crawlOnlineDocuments, nodeId, pipeline_id])
 
   useEffect(() => {
-    getNotionData()
+    getOnlineDocuments()
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
-  const firstWorkspaceId = notionData[0]?.workspace_id
+  const firstWorkspaceId = documentsData[0]?.workspace_id
-  const currentWorkspace = notionData.find(workspace => workspace.workspace_id === currentWorkspaceId)
+  const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
 
   const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
     const selectedPagesId = new Set<string>()
     const boundPagesId = new Set<string>()
-    const pagesMap = notionData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+    const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
       next.pages.forEach((page) => {
         if (page.is_bound) {
           selectedPagesId.add(page.page_id)
@@ -82,7 +82,7 @@ const NotionPageSelector = ({
       return prev
     }, {})
     return [pagesMap, selectedPagesId, boundPagesId]
-  }, [notionData])
+  }, [documentsData])
   const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])]
   const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
 
@@ -107,7 +107,7 @@ const NotionPageSelector = ({
     setCurrentWorkspaceId(firstWorkspaceId)
   }, [firstWorkspaceId])
 
-  if (!notionData?.length)
+  if (!documentsData?.length)
     return null
 
   return (
@@ -121,7 +121,7 @@ const NotionPageSelector = ({
           <div className='flex grow items-center gap-x-1'>
             <WorkspaceSelector
               value={currentWorkspaceId || firstWorkspaceId}
-              items={notionData}
+              items={documentsData}
               onSelect={handleSelectWorkspace}
             />
           </div>
@@ -148,4 +148,4 @@ const NotionPageSelector = ({
   )
 }
 
-export default NotionPageSelector
+export default OnlineDocumentSelector

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -101,16 +101,16 @@ export const useLocalFile = () => {
   }
 }
 
-export const useNotionPages = () => {
+export const useOnlineDocuments = () => {
-  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
+  const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
 
-  const updateNotionPages = (value: NotionPage[]) => {
+  const updateOnlineDocuments = (value: NotionPage[]) => {
-    setNotionPages(value)
+    setOnlineDocuments(value)
   }
 
   return {
-    notionPages,
+    onlineDocuments,
-    updateNotionPages,
+    updateOnlineDocuments,
   }
 }
 

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -1,10 +1,10 @@
 import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store'
 import { useCallback, useMemo, useState } from 'react'
-import { useLocalFile, useNotionPages, useTestRunSteps, useWebsiteCrawl } from './hooks'
+import { useLocalFile, useOnlineDocuments, useTestRunSteps, useWebsiteCrawl } from './hooks'
 import DataSourceOptions from './data-source-options'
 import LocalFile from './data-source/local-file'
 import { useProviderContextSelector } from '@/context/provider-context'
-import Notion from './data-source/notion'
+import OnlineDocuments from './data-source/online-documents'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import WebsiteCrawl from './data-source/website-crawl'
 import Actions from './data-source/actions'
@@ -35,9 +35,9 @@ const TestRunPanel = () => {
     updateFileList,
   } = useLocalFile()
   const {
-    notionPages,
+    onlineDocuments,
-    updateNotionPages,
+    updateOnlineDocuments,
-  } = useNotionPages()
+  } = useOnlineDocuments()
   const {
     websitePages,
     websiteCrawlJobId,
@@ -54,11 +54,11 @@ const TestRunPanel = () => {
     if (datasource.type === DatasourceType.localFile)
       return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
     if (datasource.type === DatasourceType.onlineDocument)
-      return isShowVectorSpaceFull || !notionPages.length
+      return isShowVectorSpaceFull || !onlineDocuments.length
     if (datasource.type === DatasourceType.websiteCrawl)
       return isShowVectorSpaceFull || !websitePages.length
     return false
-  }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
+  }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
 
   const handleClose = () => {
     setShowDebugAndPreviewPanel(false)
@@ -83,7 +83,7 @@ const TestRunPanel = () => {
       datasourceInfoList.push(documentInfo)
     }
     if (datasource.type === DatasourceType.onlineDocument) {
-      const { workspace_id, ...rest } = notionPages[0]
+      const { workspace_id, ...rest } = onlineDocuments[0]
       const documentInfo = {
         workspace_id,
         page: rest,
@@ -103,7 +103,7 @@ const TestRunPanel = () => {
       datasource_type: datasource.type,
       datasource_info_list: datasourceInfoList,
     })
-  }, [datasource, fileList, handleRun, notionPages, websiteCrawlJobId, websitePages])
+  }, [datasource, fileList, handleRun, onlineDocuments, websiteCrawlJobId, websitePages])
 
   return (
     <div
@@ -130,15 +130,15 @@ const TestRunPanel = () => {
                   />
                 )}
                 {datasource?.type === DatasourceType.onlineDocument && (
-                  <Notion
+                  <OnlineDocuments
                     nodeId={datasource?.nodeId || ''}
                     headerInfo={{
                       title: datasource.description,
                       docTitle: datasource.docTitle || '',
                       docLink: datasource.docLink || '',
                     }}
-                    notionPages={notionPages}
+                    onlineDocuments={onlineDocuments}
-                    updateNotionPages={updateNotionPages}
+                    updateOnlineDocuments={updateOnlineDocuments}
                     isInPipeline
                   />
                 )}