feat: Integrate OnlineDrive component into CreateFormPipeline and update related components

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -27,6 +27,7 @@ import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, Publis
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
 import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
+import OnlineDrive from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive'
 
 const CreateFormPipeline = () => {
   const { t } = useTranslation()
@@ -282,6 +283,11 @@ const CreateFormPipeline = () => {
                       previewIndex={previewIndex}
                     />
                   )}
+                  {datasourceType === DatasourceType.onlineDrive && (
+                    <OnlineDrive
+                      nodeData={datasource!.nodeData}
+                    />
+                  )}
                   {isShowVectorSpaceFull && (
                     <VectorSpaceFull />
                   )}

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx

@@ -1,7 +1,7 @@
 import { useCallback, useEffect, useMemo, useState } from 'react'
 import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
 import SearchInput from '@/app/components/base/notion-page-selector/search-input'
-import PageSelector from '@/app/components/base/notion-page-selector/page-selector'
+import PageSelector from './page-selector'
 import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
 import Header from '@/app/components/datasets/create/website/base/header'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx

@@ -1,14 +1,10 @@
-import { memo, useMemo, useState } from 'react'
+import { useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import { FixedSizeList as List, areEqual } from 'react-window'
+import { FixedSizeList as List } from 'react-window'
-import type { ListChildComponentProps } from 'react-window'
-import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
-import Checkbox from '@/app/components/base/checkbox'
-import NotionIcon from '@/app/components/base/notion-icon'
-import cn from '@/utils/classnames'
 import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
+import Item from './item'
+import { recursivePushInParentDescendants } from './utils'
 
-// todo: refactor this component to use the new OnlineDocumentSelector component
 type PageSelectorProps = {
   value: Set<string>
   disabledValue: Set<string>
@@ -20,169 +16,21 @@ type PageSelectorProps = {
   previewPageId?: string
   onPreview?: (selectedPageId: string) => void
 }
-type NotionPageTreeItem = {
+
+export type NotionPageTreeItem = {
   children: Set<string>
   descendants: Set<string>
   depth: number
   ancestors: string[]
 } & DataSourceNotionPage
-type NotionPageTreeMap = Record<string, NotionPageTreeItem>
+
+export type NotionPageTreeMap = Record<string, NotionPageTreeItem>
+
 type NotionPageItem = {
   expand: boolean
   depth: number
 } & DataSourceNotionPage
 
-const recursivePushInParentDescendants = (
-  pagesMap: DataSourceNotionPageMap,
-  listTreeMap: NotionPageTreeMap,
-  current: NotionPageTreeItem,
-  leafItem: NotionPageTreeItem,
-) => {
-  const parentId = current.parent_id
-  const pageId = current.page_id
-
-  if (!parentId || !pageId)
-    return
-
-  if (parentId !== 'root' && pagesMap[parentId]) {
-    if (!listTreeMap[parentId]) {
-      const children = new Set([pageId])
-      const descendants = new Set([pageId, leafItem.page_id])
-      listTreeMap[parentId] = {
-        ...pagesMap[parentId],
-        children,
-        descendants,
-        depth: 0,
-        ancestors: [],
-      }
-    }
-    else {
-      listTreeMap[parentId].children.add(pageId)
-      listTreeMap[parentId].descendants.add(pageId)
-      listTreeMap[parentId].descendants.add(leafItem.page_id)
-    }
-    leafItem.depth++
-    leafItem.ancestors.unshift(listTreeMap[parentId].page_name)
-
-    if (listTreeMap[parentId].parent_id !== 'root')
-      recursivePushInParentDescendants(pagesMap, listTreeMap, listTreeMap[parentId], leafItem)
-  }
-}
-
-const ItemComponent = ({ index, style, data }: ListChildComponentProps<{
-  dataList: NotionPageItem[]
-  handleToggle: (index: number) => void
-  checkedIds: Set<string>
-  disabledCheckedIds: Set<string>
-  handleCheck: (index: number) => void
-  canPreview?: boolean
-  handlePreview: (index: number) => void
-  listMapWithChildrenAndDescendants: NotionPageTreeMap
-  searchValue: string
-  previewPageId: string
-  pagesMap: DataSourceNotionPageMap
-}>) => {
-  const { t } = useTranslation()
-  const {
-    dataList,
-    handleToggle,
-    checkedIds,
-    disabledCheckedIds,
-    handleCheck,
-    canPreview,
-    handlePreview,
-    listMapWithChildrenAndDescendants,
-    searchValue,
-    previewPageId,
-    pagesMap,
-  } = data
-  const current = dataList[index]
-  const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id]
-  const hasChild = currentWithChildrenAndDescendants.descendants.size > 0
-  const ancestors = currentWithChildrenAndDescendants.ancestors
-  const breadCrumbs = ancestors.length ? [...ancestors, current.page_name] : [current.page_name]
-  const disabled = disabledCheckedIds.has(current.page_id)
-
-  const renderArrow = () => {
-    if (hasChild) {
-      return (
-        <div
-          className='mr-1 flex h-5 w-5 shrink-0 items-center justify-center rounded-md hover:bg-components-button-ghost-bg-hover'
-          style={{ marginLeft: current.depth * 8 }}
-          onClick={() => handleToggle(index)}
-        >
-          {
-            current.expand
-              ? <RiArrowDownSLine className='h-4 w-4 text-text-tertiary' />
-              : <RiArrowRightSLine className='h-4 w-4 text-text-tertiary' />
-          }
-        </div>
-      )
-    }
-    if (current.parent_id === 'root' || !pagesMap[current.parent_id]) {
-      return (
-        <div></div>
-      )
-    }
-    return (
-      <div className='mr-1 h-5 w-5 shrink-0' style={{ marginLeft: current.depth * 8 }} />
-    )
-  }
-
-  return (
-    <div
-      className={cn('group flex cursor-pointer items-center rounded-md pl-2 pr-[2px] hover:bg-state-base-hover',
-        previewPageId === current.page_id && 'bg-state-base-hover')}
-      style={{ ...style, top: style.top as number + 8, left: 8, right: 8, width: 'calc(100% - 16px)' }}
-    >
-      <Checkbox
-        className='mr-2 shrink-0'
-        checked={checkedIds.has(current.page_id)}
-        disabled={disabled}
-        onCheck={() => {
-          if (disabled)
-            return
-          handleCheck(index)
-        }}
-      />
-      {!searchValue && renderArrow()}
-      <NotionIcon
-        className='mr-1 shrink-0'
-        type='page'
-        src={current.page_icon}
-      />
-      <div
-        className='grow truncate text-[13px] font-medium leading-4 text-text-secondary'
-        title={current.page_name}
-      >
-        {current.page_name}
-      </div>
-      {
-        canPreview && (
-          <div
-            className='ml-1 hidden h-6 shrink-0 cursor-pointer items-center rounded-md border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-2 text-xs
-            font-medium leading-4 text-components-button-secondary-text shadow-xs shadow-shadow-shadow-3 backdrop-blur-[10px]
-            hover:border-components-button-secondary-border-hover hover:bg-components-button-secondary-bg-hover group-hover:flex'
-            onClick={() => handlePreview(index)}>
-            {t('common.dataSource.notion.selector.preview')}
-          </div>
-        )
-      }
-      {
-        searchValue && (
-          <div
-            className='ml-1 max-w-[120px] shrink-0 truncate text-xs text-text-quaternary'
-            title={breadCrumbs.join(' / ')}
-          >
-            {breadCrumbs.join(' / ')}
-          </div>
-        )
-      }
-    </div>
-  )
-}
-const Item = memo(ItemComponent, areEqual)
-
 const PageSelector = ({
   value,
   disabledValue,

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/item.tsx

@@ -0,0 +1,138 @@
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { areEqual } from 'react-window'
+import type { ListChildComponentProps } from 'react-window'
+import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
+import Checkbox from '@/app/components/base/checkbox'
+import NotionIcon from '@/app/components/base/notion-icon'
+import cn from '@/utils/classnames'
+import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
+
+type NotionPageTreeItem = {
+  children: Set<string>
+  descendants: Set<string>
+  depth: number
+  ancestors: string[]
+} & DataSourceNotionPage
+
+type NotionPageTreeMap = Record<string, NotionPageTreeItem>
+
+type NotionPageItem = {
+  expand: boolean
+  depth: number
+} & DataSourceNotionPage
+
+const Item = ({ index, style, data }: ListChildComponentProps<{
+  dataList: NotionPageItem[]
+  handleToggle: (index: number) => void
+  checkedIds: Set<string>
+  disabledCheckedIds: Set<string>
+  handleCheck: (index: number) => void
+  canPreview?: boolean
+  handlePreview: (index: number) => void
+  listMapWithChildrenAndDescendants: NotionPageTreeMap
+  searchValue: string
+  previewPageId: string
+  pagesMap: DataSourceNotionPageMap
+}>) => {
+  const { t } = useTranslation()
+  const {
+    dataList,
+    handleToggle,
+    checkedIds,
+    disabledCheckedIds,
+    handleCheck,
+    canPreview,
+    handlePreview,
+    listMapWithChildrenAndDescendants,
+    searchValue,
+    previewPageId,
+    pagesMap,
+  } = data
+  const current = dataList[index]
+  const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id]
+  const hasChild = currentWithChildrenAndDescendants.descendants.size > 0
+  const ancestors = currentWithChildrenAndDescendants.ancestors
+  const breadCrumbs = ancestors.length ? [...ancestors, current.page_name] : [current.page_name]
+  const disabled = disabledCheckedIds.has(current.page_id)
+
+  const renderArrow = () => {
+    if (hasChild) {
+      return (
+        <div
+          className='mr-1 flex h-5 w-5 shrink-0 items-center justify-center rounded-md hover:bg-components-button-ghost-bg-hover'
+          style={{ marginLeft: current.depth * 8 }}
+          onClick={() => handleToggle(index)}
+        >
+          {
+            current.expand
+              ? <RiArrowDownSLine className='h-4 w-4 text-text-tertiary' />
+              : <RiArrowRightSLine className='h-4 w-4 text-text-tertiary' />
+          }
+        </div>
+      )
+    }
+    if (current.parent_id === 'root' || !pagesMap[current.parent_id]) {
+      return (
+        <div></div>
+      )
+    }
+    return (
+      <div className='mr-1 h-5 w-5 shrink-0' style={{ marginLeft: current.depth * 8 }} />
+    )
+  }
+
+  return (
+    <div
+      className={cn('group flex cursor-pointer items-center rounded-md pl-2 pr-[2px] hover:bg-state-base-hover',
+        previewPageId === current.page_id && 'bg-state-base-hover')}
+      style={{ ...style, top: style.top as number + 8, left: 8, right: 8, width: 'calc(100% - 16px)' }}
+    >
+      <Checkbox
+        className='mr-2 shrink-0'
+        checked={checkedIds.has(current.page_id)}
+        disabled={disabled}
+        onCheck={() => {
+          if (disabled)
+            return
+          handleCheck(index)
+        }}
+      />
+      {!searchValue && renderArrow()}
+      <NotionIcon
+        className='mr-1 shrink-0'
+        type='page'
+        src={current.page_icon}
+      />
+      <div
+        className='grow truncate text-[13px] font-medium leading-4 text-text-secondary'
+        title={current.page_name}
+      >
+        {current.page_name}
+      </div>
+      {
+        canPreview && (
+          <div
+            className='ml-1 hidden h-6 shrink-0 cursor-pointer items-center rounded-md border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-2 text-xs
+            font-medium leading-4 text-components-button-secondary-text shadow-xs shadow-shadow-shadow-3 backdrop-blur-[10px]
+            hover:border-components-button-secondary-border-hover hover:bg-components-button-secondary-bg-hover group-hover:flex'
+            onClick={() => handlePreview(index)}>
+            {t('common.dataSource.notion.selector.preview')}
+          </div>
+        )
+      }
+      {
+        searchValue && (
+          <div
+            className='ml-1 max-w-[120px] shrink-0 truncate text-xs text-text-quaternary'
+            title={breadCrumbs.join(' / ')}
+          >
+            {breadCrumbs.join(' / ')}
+          </div>
+        )
+      }
+    </div>
+  )
+}
+
+export default React.memo(Item, areEqual)

web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/utils.ts

@@ -0,0 +1,39 @@
+import type { DataSourceNotionPageMap } from '@/models/common'
+import type { NotionPageTreeItem, NotionPageTreeMap } from './index'
+
+export const recursivePushInParentDescendants = (
+  pagesMap: DataSourceNotionPageMap,
+  listTreeMap: NotionPageTreeMap,
+  current: NotionPageTreeItem,
+  leafItem: NotionPageTreeItem,
+) => {
+  const parentId = current.parent_id
+  const pageId = current.page_id
+
+  if (!parentId || !pageId)
+    return
+
+  if (parentId !== 'root' && pagesMap[parentId]) {
+    if (!listTreeMap[parentId]) {
+      const children = new Set([pageId])
+      const descendants = new Set([pageId, leafItem.page_id])
+      listTreeMap[parentId] = {
+        ...pagesMap[parentId],
+        children,
+        descendants,
+        depth: 0,
+        ancestors: [],
+      }
+    }
+    else {
+      listTreeMap[parentId].children.add(pageId)
+      listTreeMap[parentId].descendants.add(pageId)
+      listTreeMap[parentId].descendants.add(leafItem.page_id)
+    }
+    leafItem.depth++
+    leafItem.ancestors.unshift(listTreeMap[parentId].page_name)
+
+    if (listTreeMap[parentId].parent_id !== 'root')
+      recursivePushInParentDescendants(pagesMap, listTreeMap, listTreeMap[parentId], leafItem)
+  }
+}

web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx

@@ -169,7 +169,7 @@ const WebsiteCrawl = ({
               usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
               previewIndex={previewIndex}
               onPreview={onPreview}
-              isMultipleChoice={false} // only support single choice in test run
+              isMultipleChoice={isInPipeline} // only support single choice in test run
             />
           )}
         </div>

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -149,8 +149,7 @@ const TestRunPanel = () => {
                   <OnlineDrive
                     nodeData={datasource!.nodeData}
                   />
-                )
+                )}
-                }
               </div>
               <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
               <FooterTips />