feat: Refactor OnlineDocuments and PageSelector components to enhance state management and integrate new Actions component

web/app/components/base/notion-page-selector/base.tsx

@@ -15,7 +15,6 @@ type NotionPageSelectorProps = {
   previewPageId?: string
   onPreview?: (selectedPage: NotionPage) => void
   datasetId?: string
-  isInPipeline?: boolean
 }
 
 const NotionPageSelector = ({
@@ -25,7 +24,6 @@ const NotionPageSelector = ({
   previewPageId,
   onPreview,
   datasetId = '',
-  isInPipeline = false,
 }: NotionPageSelectorProps) => {
   const { data, refetch } = usePreImportNotionPages({ url: '/notion/pre-import/pages', datasetId })
   const [prevData, setPrevData] = useState(data)
@@ -98,7 +96,6 @@ const NotionPageSelector = ({
           ? (
             <div className='flex flex-col gap-y-2'>
               <Header
-                isInPipeline={isInPipeline}
                 onClickConfiguration={handleConfigureNotion}
                 title={'Choose notion pages'}
                 buttonText={'Configure Notion'}

web/app/components/datasets/documents/create-from-pipeline/actions/index.tsx

@@ -8,6 +8,7 @@ import Link from 'next/link'
 type ActionsProps = {
   disabled?: boolean
   handleNextStep: () => void
+  showSelect?: boolean
 }
 
 const Actions = ({

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -7,7 +7,7 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so
 import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets'
 import { CrawlStep } from '@/models/datasets'
 import produce from 'immer'
-import type { NotionPage } from '@/models/common'
+import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
 
 export const useAddDocumentsSteps = () => {
   const { t } = useTranslation()
@@ -128,9 +128,34 @@ export const useLocalFile = () => {
 }
 
 export const useOnlineDocuments = () => {
+  const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
+  const [searchValue, setSearchValue] = useState('')
+  const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
   const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
   const [currentDocument, setCurrentDocument] = useState<NotionPage | undefined>()
 
+  const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
+    const selectedPagesId = new Set<string>()
+    const boundPagesId = new Set<string>()
+    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+      next.pages.forEach((page) => {
+        if (page.is_bound) {
+          selectedPagesId.add(page.page_id)
+          boundPagesId.add(page.page_id)
+        }
+        prev[page.page_id] = {
+          ...page,
+          workspace_id: next.workspace_id,
+        }
+      })
+
+      return prev
+    }, {})
+    return [pagesMap, selectedPagesId, boundPagesId]
+  }, [documentsData])
+  const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(onlineDocuments.map(doc => doc.page_id) || [])]
+  const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
+
   const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])
 
   const updateOnlineDocuments = (value: NotionPage[]) => {
@@ -146,6 +171,15 @@ export const useOnlineDocuments = () => {
   }, [])
 
   return {
+    documentsData,
+    setDocumentsData,
+    searchValue,
+    setSearchValue,
+    currentWorkspaceId,
+    setCurrentWorkspaceId,
+    PagesMapAndSelectedPagesId,
+    selectedPagesId,
+    setSelectedPagesId,
     onlineDocuments,
     previewOnlineDocument,
     updateOnlineDocuments,

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -8,7 +8,7 @@ import type { NotionPage } from '@/models/common'
 import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl'
-import Actions from './data-source/actions'
+import Actions from './actions'
 import { useTranslation } from 'react-i18next'
 import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
 import LeftHeader from './left-header'
@@ -61,6 +61,15 @@ const CreateFormPipeline = () => {
     hideFilePreview,
   } = useLocalFile()
   const {
+    documentsData,
+    setDocumentsData,
+    searchValue,
+    setSearchValue,
+    currentWorkspaceId,
+    setCurrentWorkspaceId,
+    PagesMapAndSelectedPagesId,
+    selectedPagesId,
+    setSelectedPagesId,
     onlineDocuments,
     previewOnlineDocument,
     updateOnlineDocuments,
@@ -261,9 +270,17 @@ const CreateFormPipeline = () => {
                   )}
                   {datasourceType === DatasourceType.onlineDocument && (
                     <OnlineDocuments
+                      documentsData={documentsData}
+                      setDocumentsData={setDocumentsData}
+                      searchValue={searchValue}
+                      setSearchValue={setSearchValue}
+                      currentWorkspaceId={currentWorkspaceId}
+                      setCurrentWorkspaceId={setCurrentWorkspaceId}
+                      PagesMapAndSelectedPagesId={PagesMapAndSelectedPagesId}
+                      selectedPagesId={selectedPagesId}
+                      setSelectedPagesId={setSelectedPagesId}
                       nodeId={datasource!.nodeId}
                       nodeData={datasource!.nodeData}
-                      pageIdList={onlineDocuments.map(doc => doc.page_id)}
                       onSelect={updateOnlineDocuments}
                       onPreview={updateCurrentPage}
                     />

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useCallback, useEffect, useMemo } from 'react'
 import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
 import SearchInput from '@/app/components/base/notion-page-selector/search-input'
 import PageSelector from './page-selector'
@@ -12,28 +12,41 @@ import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 
 type OnlineDocumentsProps = {
-  pageIdList?: string[]
   onSelect: (selectedPages: NotionPage[]) => void
   previewPageId?: string
   onPreview?: (selectedPage: NotionPage) => void
   isInPipeline?: boolean
   nodeId: string
   nodeData: DataSourceNodeType
+  documentsData: DataSourceNotionWorkspace[]
+  setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => void
+  searchValue: string
+  setSearchValue: (value: string) => void
+  currentWorkspaceId: string
+  setCurrentWorkspaceId: (workspaceId: string) => void
+  PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>]
+  selectedPagesId: Set<string>
+  setSelectedPagesId: (selectedPagesId: Set<string>) => void
 }
 
 const OnlineDocuments = ({
-  pageIdList,
   onSelect,
   previewPageId,
   onPreview,
   isInPipeline = false,
   nodeId,
   nodeData,
+  documentsData,
+  setDocumentsData,
+  searchValue,
+  setSearchValue,
+  currentWorkspaceId,
+  setCurrentWorkspaceId,
+  PagesMapAndSelectedPagesId,
+  selectedPagesId,
+  setSelectedPagesId,
 }: OnlineDocumentsProps) => {
   const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
-  const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
-  const [searchValue, setSearchValue] = useState('')
-  const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
 
   const datasourceNodeRunURL = !isInPipeline
     ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
@@ -51,6 +64,7 @@ const OnlineDocuments = ({
       {
         onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
           setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
+          setCurrentWorkspaceId(documentsData.data[0].workspace_id)
         },
         onError: (message: string) => {
           Toast.notify({
@@ -60,62 +74,36 @@ const OnlineDocuments = ({
         },
       },
     )
-  }, [datasourceNodeRunURL])
+  }, [datasourceNodeRunURL, setCurrentWorkspaceId, setDocumentsData])
 
   useEffect(() => {
-    getOnlineDocuments()
+    if (!documentsData.length)
+      getOnlineDocuments()
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
-  const firstWorkspaceId = documentsData[0]?.workspace_id
   const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
 
-  const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
-    const selectedPagesId = new Set<string>()
-    const boundPagesId = new Set<string>()
-    const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
-      next.pages.forEach((page) => {
-        if (page.is_bound) {
-          selectedPagesId.add(page.page_id)
-          boundPagesId.add(page.page_id)
-        }
-        prev[page.page_id] = {
-          ...page,
-          workspace_id: next.workspace_id,
-        }
-      })
-
-      return prev
-    }, {})
-    return [pagesMap, selectedPagesId, boundPagesId]
-  }, [documentsData])
-  const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(pageIdList || [])]
-  const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
-
   const handleSearchValueChange = useCallback((value: string) => {
     setSearchValue(value)
-  }, [])
+  }, [setSearchValue])
 
   const handleSelectWorkspace = useCallback((workspaceId: string) => {
     setCurrentWorkspaceId(workspaceId)
-  }, [])
+  }, [setCurrentWorkspaceId])
 
   const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
     const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId])
 
     setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
     onSelect(selectedPages)
-  }, [onSelect, PagesMapAndSelectedPagesId])
+  }, [setSelectedPagesId, onSelect, PagesMapAndSelectedPagesId])
 
   const handlePreviewPage = useCallback((previewPageId: string) => {
     if (onPreview)
       onPreview(PagesMapAndSelectedPagesId[0][previewPageId])
   }, [PagesMapAndSelectedPagesId, onPreview])
 
-  useEffect(() => {
-    setCurrentWorkspaceId(firstWorkspaceId)
-  }, [firstWorkspaceId])
-
   const headerInfo = useMemo(() => {
     return {
       title: nodeData.title,
@@ -137,7 +125,7 @@ const OnlineDocuments = ({
         <div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
           <div className='flex grow items-center gap-x-1'>
             <WorkspaceSelector
-              value={currentWorkspaceId || firstWorkspaceId}
+              value={currentWorkspaceId}
               items={documentsData}
               onSelect={handleSelectWorkspace}
             />
@@ -149,7 +137,7 @@ const OnlineDocuments = ({
         </div>
         <div className='overflow-hidden rounded-b-xl'>
           <PageSelector
-            value={selectedPagesId}
+            checkedIds={selectedPagesId}
             disabledValue={PagesMapAndSelectedPagesId[2]}
             searchValue={searchValue}
             list={currentWorkspace?.pages || []}
@@ -159,6 +147,7 @@ const OnlineDocuments = ({
             previewPageId={previewPageId}
             onPreview={handlePreviewPage}
             isMultipleChoice={!isInPipeline}
+            currentWorkspaceId={currentWorkspaceId}
           />
         </div>
       </div>

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx

@@ -6,7 +6,7 @@ import Item from './item'
 import { recursivePushInParentDescendants } from './utils'
 
 type PageSelectorProps = {
-  value: Set<string>
+  checkedIds: Set<string>
   disabledValue: Set<string>
   searchValue: string
   pagesMap: DataSourceNotionPageMap
@@ -16,6 +16,7 @@ type PageSelectorProps = {
   previewPageId?: string
   onPreview?: (selectedPageId: string) => void
   isMultipleChoice?: boolean
+  currentWorkspaceId: string
 }
 
 export type NotionPageTreeItem = {
@@ -33,7 +34,7 @@ type NotionPageItem = {
 } & DataSourceNotionPage
 
 const PageSelector = ({
-  value,
+  checkedIds,
   disabledValue,
   searchValue,
   pagesMap,
@@ -43,24 +44,22 @@ const PageSelector = ({
   previewPageId,
   onPreview,
   isMultipleChoice = true,
+  currentWorkspaceId,
 }: PageSelectorProps) => {
   const { t } = useTranslation()
-  const [prevDataList, setPrevDataList] = useState(list)
   const [dataList, setDataList] = useState<NotionPageItem[]>([])
   const [localPreviewPageId, setLocalPreviewPageId] = useState('')
 
   useEffect(() => {
-    if (prevDataList !== list) {
+    setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
-      setPrevDataList(list)
+      return {
-      setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
+        ...item,
-        return {
+        expand: false,
-          ...item,
+        depth: 0,
-          expand: false,
+      }
-          depth: 0,
+    }))
-        }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
-      }))
+  }, [currentWorkspaceId])
-    }
-  }, [prevDataList, list, pagesMap])
 
   const searchDataList = list.filter((item) => {
     return item.page_name.includes(searchValue)
@@ -108,13 +107,14 @@ const PageSelector = ({
           expand: false,
           depth: listMapWithChildrenAndDescendants[item].depth,
         })),
-        ...dataList.slice(index + 1)]
+        ...dataList.slice(index + 1),
+      ]
     }
     setDataList(newDataList)
   }, [dataList, listMapWithChildrenAndDescendants, pagesMap])
 
   const handleCheck = useCallback((index: number) => {
-    const copyValue = new Set([...value])
+    const copyValue = new Set([...checkedIds])
     const current = currentDataList[index]
     const pageId = current.page_id
     const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
@@ -143,7 +143,7 @@ const PageSelector = ({
     }
 
     onSelect(new Set([...copyValue]))
-  }, [currentDataList, isMultipleChoice, listMapWithChildrenAndDescendants, onSelect, searchValue, value])
+  }, [currentDataList, isMultipleChoice, listMapWithChildrenAndDescendants, onSelect, searchValue, checkedIds])
 
   const handlePreview = useCallback((index: number) => {
     const current = currentDataList[index]
@@ -174,7 +174,7 @@ const PageSelector = ({
       itemData={{
         dataList: currentDataList,
         handleToggle,
-        checkedIds: value,
+        checkedIds,
         disabledCheckedIds: disabledValue,
         handleCheck,
         canPreview,

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -8,7 +8,7 @@ import { useCallback, useMemo, useState } from 'react'
 import type { CrawlResult } from '@/models/datasets'
 import { type CrawlResultItem, CrawlStep, type FileItem } from '@/models/datasets'
 import produce from 'immer'
-import type { NotionPage } from '@/models/common'
+import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
 
 export const useTestRunSteps = () => {
   const { t } = useTranslation()
@@ -107,13 +107,47 @@ export const useLocalFile = () => {
 }
 
 export const useOnlineDocuments = () => {
+  const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
+  const [searchValue, setSearchValue] = useState('')
+  const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
   const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
 
+  const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
+    const selectedPagesId = new Set<string>()
+    const boundPagesId = new Set<string>()
+    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+      next.pages.forEach((page) => {
+        if (page.is_bound) {
+          selectedPagesId.add(page.page_id)
+          boundPagesId.add(page.page_id)
+        }
+        prev[page.page_id] = {
+          ...page,
+          workspace_id: next.workspace_id,
+        }
+      })
+
+      return prev
+    }, {})
+    return [pagesMap, selectedPagesId, boundPagesId]
+  }, [documentsData])
+  const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(onlineDocuments.map(doc => doc.page_id) || [])]
+  const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
+
   const updateOnlineDocuments = (value: NotionPage[]) => {
     setOnlineDocuments(value)
   }
 
   return {
+    documentsData,
+    setDocumentsData,
+    searchValue,
+    setSearchValue,
+    currentWorkspaceId,
+    setCurrentWorkspaceId,
+    PagesMapAndSelectedPagesId,
+    selectedPagesId,
+    setSelectedPagesId,
     onlineDocuments,
     updateOnlineDocuments,
   }

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -32,6 +32,15 @@ const TestRunPanel = () => {
     updateFileList,
   } = useLocalFile()
   const {
+    documentsData,
+    setDocumentsData,
+    searchValue,
+    setSearchValue,
+    currentWorkspaceId,
+    setCurrentWorkspaceId,
+    PagesMapAndSelectedPagesId,
+    selectedPagesId,
+    setSelectedPagesId,
     onlineDocuments,
     updateOnlineDocuments,
   } = useOnlineDocuments()
@@ -125,9 +134,17 @@ const TestRunPanel = () => {
                 )}
                 {datasourceType === DatasourceType.onlineDocument && (
                   <OnlineDocuments
+                    documentsData={documentsData}
+                    setDocumentsData={setDocumentsData}
+                    searchValue={searchValue}
+                    setSearchValue={setSearchValue}
+                    currentWorkspaceId={currentWorkspaceId}
+                    setCurrentWorkspaceId={setCurrentWorkspaceId}
+                    PagesMapAndSelectedPagesId={PagesMapAndSelectedPagesId}
+                    selectedPagesId={selectedPagesId}
+                    setSelectedPagesId={setSelectedPagesId}
                     nodeId={datasource!.nodeId}
                     nodeData={datasource!.nodeData}
-                    pageIdList={onlineDocuments.map(doc => doc.page_id)}
                     onSelect={updateOnlineDocuments}
                     isInPipeline
                   />