feat: Enhance CreateFormPipeline with file selection and validation for online documents and drives

web/app/components/datasets/documents/create-from-pipeline/actions/index.tsx

@@ -1,45 +1,88 @@
-import React from 'react'
+import React, { useMemo } from 'react'
 import Button from '@/app/components/base/button'
 import { useTranslation } from 'react-i18next'
 import { useParams } from 'next/navigation'
 import { RiArrowRightLine } from '@remixicon/react'
 import Link from 'next/link'
+import Checkbox from '@/app/components/base/checkbox'
 
 type ActionsProps = {
   disabled?: boolean
   handleNextStep: () => void
   showSelect?: boolean
+  totalOptions?: number
+  selectedOptions?: number
+  onSelectAll?: () => void
+  tip?: string
 }
 
 const Actions = ({
   disabled,
   handleNextStep,
+  showSelect = false,
+  totalOptions,
+  selectedOptions,
+  onSelectAll,
+  tip = '',
 }: ActionsProps) => {
   const { t } = useTranslation()
   const { datasetId } = useParams()
 
+  const indeterminate = useMemo(() => {
+    if (!showSelect) return false
+    if (selectedOptions === undefined || totalOptions === undefined) return false
+    return selectedOptions > 0 && selectedOptions < totalOptions
+  }, [showSelect, selectedOptions, totalOptions])
+
+  const checked = useMemo(() => {
+    if (!showSelect) return false
+    if (selectedOptions === undefined || totalOptions === undefined) return false
+    return selectedOptions > 0 && selectedOptions === totalOptions
+  }, [showSelect, selectedOptions, totalOptions])
+
   return (
-    <div className='flex justify-end gap-x-2'>
-      <Link
-        href={`/datasets/${datasetId}/documents`}
-        replace
-      >
+    <div className='flex items-center gap-x-2'>
+      {showSelect && (
+        <>
+          <div className='flex shrink-0 items-center gap-x-2 py-[3px] pl-4 pr-2'>
+            <Checkbox
+              onCheck={onSelectAll}
+              indeterminate={indeterminate}
+              checked={checked}
+            />
+            <span className='system-sm-medium text-text-accent'>
+              {t('common.operation.selectAll')}
+            </span>
+          </div>
+          {tip && (
+            <div className='system-xs-regular shrink-0 text-text-tertiary'>
+              {tip}
+            </div>
+          )}
+        </>
+      )}
+      <div className='flex grow items-center justify-end gap-x-2'>
+        <Link
+          href={`/datasets/${datasetId}/documents`}
+          replace
+        >
+          <Button
+            variant='ghost'
+            className='px-3 py-2'
+          >
+            {t('common.operation.cancel')}
+          </Button>
+        </Link>
         <Button
-          variant='ghost'
-          className='px-3 py-2'
+          disabled={disabled}
+          variant='primary'
+          onClick={handleNextStep}
+          className='gap-x-0.5'
         >
-          {t('common.operation.cancel')}
+          <span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
+          <RiArrowRightLine className='size-4' />
         </Button>
-      </Link>
-      <Button
-        disabled={disabled}
-        variant='primary'
-        onClick={handleNextStep}
-        className='gap-x-0.5'
-      >
-        <span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
-        <RiArrowRightLine className='size-4' />
-      </Button>
+      </div>
     </div>
   )
 }

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx

@@ -1,12 +1,11 @@
 import React, { useEffect, useMemo, useRef } from 'react'
-import type { OnlineDriveFile } from '@/models/pipeline'
+import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline'
 import Item from './item'
 import EmptyFolder from './empty-folder'
 import EmptySearchResult from './empty-search-result'
 import Loading from '@/app/components/base/loading'
 import { RiLoader2Line } from '@remixicon/react'
 import { useFileSupportTypes } from '@/service/use-common'
-import { isFile } from '../../utils'
 import { getFileExtension } from './utils'
 import { useDataSourceStore } from '../../../store'
 
@@ -84,7 +83,7 @@ const List = ({
             fileList.map((file) => {
               const isSelected = selectedFileList.includes(file.key)
               const extension = getFileExtension(file.key)
-              const disabled = isFile(file.key) && !supportedFileTypes.includes(extension)
+              const disabled = file.type === OnlineDriveFileType.file && !supportedFileTypes.includes(extension)
               return (
                 <Item
                   key={file.key}

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -5,6 +5,7 @@ import { useCallback, useMemo, useState } from 'react'
 import { BlockEnum, type Node } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 import { useDataSourceStore, useDataSourceStoreWithSelector } from './data-source/store'
+import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
 
 export const useAddDocumentsSteps = () => {
   const { t } = useTranslation()
@@ -83,19 +84,39 @@ export const useLocalFile = () => {
 }
 
 export const useOnlineDocuments = () => {
+  const documentsData = useDataSourceStoreWithSelector(state => state.documentsData)
+  const currentWorkspaceId = useDataSourceStoreWithSelector(state => state.currentWorkspaceId)
   const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
   const previewOnlineDocumentRef = useDataSourceStoreWithSelector(state => state.previewOnlineDocumentRef)
   const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument)
   const dataSourceStore = useDataSourceStore()
 
+  const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
+
+  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
+    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+      next.pages.forEach((page) => {
+        prev[page.page_id] = {
+          ...page,
+          workspace_id: next.workspace_id,
+        }
+      })
+
+      return prev
+    }, {})
+    return pagesMap
+  }, [documentsData])
+
   const hidePreviewOnlineDocument = useCallback(() => {
     const { setCurrentDocument } = dataSourceStore.getState()
     setCurrentDocument(undefined)
   }, [dataSourceStore])
 
   return {
+    currentWorkspace,
     onlineDocuments,
     currentDocument,
+    PagesMapAndSelectedPagesId,
     previewOnlineDocumentRef,
     hidePreviewOnlineDocument,
   }

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -30,6 +30,8 @@ import { TransferMethod } from '@/types/app'
 import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
 import DataSourceProvider from './data-source/store/provider'
 import { useDataSourceStore } from './data-source/store'
+import { useFileSupportTypes, useFileUploadConfig } from '@/service/use-common'
+import { getFileExtension } from './data-source/online-drive/file-list/list/utils'
 
 const CreateFormPipeline = () => {
   const { t } = useTranslation()
@@ -46,6 +48,8 @@ const CreateFormPipeline = () => {
   const formRef = useRef<any>(null)
 
   const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')
+  const { data: fileUploadConfigResponse } = useFileUploadConfig()
+  const { data: supportFileTypesRes } = useFileSupportTypes()
 
   const {
     steps,
@@ -61,8 +65,10 @@ const CreateFormPipeline = () => {
     hidePreviewLocalFile,
   } = useLocalFile()
   const {
+    currentWorkspace,
     onlineDocuments,
     currentDocument,
+    PagesMapAndSelectedPagesId,
     previewOnlineDocumentRef,
     hidePreviewOnlineDocument,
   } = useOnlineDocuments()
@@ -107,6 +113,60 @@ const CreateFormPipeline = () => {
     return false
   }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileList.length])
 
+  const showSelect = useMemo(() => {
+    if (datasourceType === DatasourceType.onlineDocument) {
+      const pagesCount = currentWorkspace?.pages.length ?? 0
+      return pagesCount > 0
+    }
+    if (datasourceType === DatasourceType.onlineDrive) {
+      const isBucketList = onlineDriveFileList.some(file => file.type === 'bucket')
+      return !isBucketList && onlineDriveFileList.length > 0
+    }
+  }, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList])
+
+  const supportedFileTypes = useMemo(() => {
+    if (!supportFileTypesRes) return []
+    return Array.from(new Set(supportFileTypesRes.allowed_extensions.map(item => item.toLowerCase())))
+  }, [supportFileTypesRes])
+
+  const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
+    file_size_limit: 15,
+    batch_count_limit: 5,
+  }, [fileUploadConfigResponse])
+
+  const totalOptions = useMemo(() => {
+    if (datasourceType === DatasourceType.onlineDocument)
+      return currentWorkspace?.pages.length
+    if (datasourceType === DatasourceType.onlineDrive) {
+      return onlineDriveFileList.filter((item) => {
+        if (item.type === 'bucket') return false
+        if (item.type === 'folder') return true
+        if (item.type === 'file')
+          return supportedFileTypes.includes(getFileExtension(item.key))
+        return false
+      }).length
+    }
+  }, [currentWorkspace?.pages.length, datasourceType, onlineDriveFileList, supportedFileTypes])
+
+  const selectedOptions = useMemo(() => {
+    if (datasourceType === DatasourceType.onlineDocument)
+      return onlineDocuments.length
+    if (datasourceType === DatasourceType.onlineDrive)
+      return selectedFileList.length
+  }, [datasourceType, onlineDocuments.length, selectedFileList.length])
+
+  const tip = useMemo(() => {
+    if (datasourceType === DatasourceType.onlineDocument)
+      return t('datasetPipeline.addDocuments.selectOnlineDocumentTip', { count: 50 })
+    if (datasourceType === DatasourceType.onlineDrive) {
+      return t('datasetPipeline.addDocuments.selectOnlineDriveTip', {
+        count: fileUploadConfig.batch_count_limit,
+        fileSize: fileUploadConfig.file_size_limit,
+      })
+    }
+    return ''
+  }, [datasourceType, fileUploadConfig.batch_count_limit, fileUploadConfig.file_size_limit, t])
+
   const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
 
   const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
@@ -242,6 +302,35 @@ const CreateFormPipeline = () => {
     onClickPreview()
   }, [onClickPreview, previewWebsitePageRef])
 
+  const handleSelectAll = useCallback(() => {
+    const { setOnlineDocuments, setSelectedFileList, setSelectedPagesId } = dataSourceStore.getState()
+    if (datasourceType === DatasourceType.onlineDocument) {
+      const allIds = currentWorkspace?.pages.map(page => page.page_id) || []
+      if (onlineDocuments.length < allIds.length) {
+        const selectedPages = Array.from(allIds).map(pageId => PagesMapAndSelectedPagesId[pageId])
+        setOnlineDocuments(selectedPages)
+        setSelectedPagesId(new Set(allIds))
+      }
+      else {
+        setOnlineDocuments([])
+        setSelectedPagesId(new Set())
+      }
+    }
+    if (datasourceType === DatasourceType.onlineDrive) {
+      const allKeys = onlineDriveFileList.filter((item) => {
+        if (item.type === 'bucket') return false
+        if (item.type === 'folder') return true
+        if (item.type === 'file')
+          return supportedFileTypes.includes(getFileExtension(item.key))
+        return false
+      }).map(file => file.key)
+      if (selectedFileList.length < allKeys.length)
+        setSelectedFileList(allKeys)
+      else
+        setSelectedFileList([])
+    }
+  }, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType, onlineDocuments.length, onlineDriveFileList, selectedFileList.length, supportedFileTypes])
+
   if (isFetchingPipelineInfo) {
     return (
       <Loading type='app' />
@@ -295,7 +384,15 @@ const CreateFormPipeline = () => {
                   {isShowVectorSpaceFull && (
                     <VectorSpaceFull />
                   )}
-                  <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
+                  <Actions
+                    showSelect={showSelect}
+                    totalOptions={totalOptions}
+                    selectedOptions={selectedOptions}
+                    onSelectAll={handleSelectAll}
+                    disabled={nextBtnDisabled}
+                    handleNextStep={handleNextStep}
+                    tip={tip}
+                  />
                 </div>
               )
             }

web/i18n/en-US/dataset-pipeline.ts

@@ -106,6 +106,8 @@ const translation = {
       learnMore: 'Learn more',
     },
     characters: 'characters',
+    selectOnlineDocumentTip: 'Process up to {{count}} pages',
+    selectOnlineDriveTip: 'Process up to {{count}} files, maximum {{fileSize}} MB each',
   },
   documentSettings: {
     title: 'Document Settings',

web/i18n/zh-Hans/dataset-pipeline.ts

@@ -106,6 +106,8 @@ const translation = {
       learnMore: '了解更多',
     },
     characters: '字符',
+    selectOnlineDocumentTip: '最多处理 {{count}} 页',
+    selectOnlineDriveTip: '最多处理 {{count}} 个文件，每个文件最大 {{fileSize}} MB',
   },
   documentSettings: {
     title: '文档设置',

web/types/pipeline.tsx

@@ -23,7 +23,7 @@ export type OnlineDriveData = {
 
 export type DataSourceNodeCompletedResponse = {
   event: 'datasource_completed'
-  data: OnlineDriveData[]
+  data: any
   time_consuming: number
 }