ngskcloud
/
gcgj-dify-1.7.0
17
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: rename selectedFileList to selectedFileKeys across components and update related logic

Browse Source
feat/rag-2
twwu 11 months ago
parent e66c2badda
commit 15cd9e0b12
13 changed files with 156 additions and 86 deletions
Show Stats Download Patch File Download Diff File

14
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
Unescape Escape View File

@ -51,9 +51,9 @@ const Breadcrumbs = ({
}, [displayBreadcrumbNum, prefix]) }, [displayBreadcrumbNum, prefix])
const handleBackToBucketList = useCallback(() => { const handleBackToBucketList = useCallback(() => {
const { setFileList, setSelectedFileList, setPrefix, setBucket } = dataSourceStore.getState() const { setFileList, setSelectedFileKeys, setPrefix, setBucket } = dataSourceStore.getState()
setFileList([]) setFileList([])
setSelectedFileList([]) setSelectedFileKeys([])
setBucket('') setBucket('')
setPrefix([]) setPrefix([])
getOnlineDriveFiles({ getOnlineDriveFiles({
@ -64,9 +64,9 @@ const Breadcrumbs = ({
}, [dataSourceStore, getOnlineDriveFiles]) }, [dataSourceStore, getOnlineDriveFiles])
const handleClickBucketName = useCallback(() => { const handleClickBucketName = useCallback(() => {
const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState() const { setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState()
setFileList([]) setFileList([])
setSelectedFileList([]) setSelectedFileKeys([])
setPrefix([]) setPrefix([])
getOnlineDriveFiles({ getOnlineDriveFiles({
prefix: [], prefix: [],
@ -75,16 +75,16 @@ const Breadcrumbs = ({
}, [dataSourceStore, getOnlineDriveFiles]) }, [dataSourceStore, getOnlineDriveFiles])
const handleClickBreadcrumb = useCallback((index: number) => { const handleClickBreadcrumb = useCallback((index: number) => {
const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState() const { prefix, setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState()
const newPrefix = prefix.slice(0, index + 1) const newPrefix = prefix.slice(0, index + 1)
setFileList([]) setFileList([])
setSelectedFileList([]) setSelectedFileKeys([])
setPrefix(newPrefix) setPrefix(newPrefix)
getOnlineDriveFiles({ getOnlineDriveFiles({
prefix: newPrefix, prefix: newPrefix,
fileList: [], fileList: [],
}) })
}, [dataSourceStore, getOnlineDriveFiles, prefix]) }, [dataSourceStore, getOnlineDriveFiles])
return ( return (
<div className='flex grow items-center overflow-hidden'> <div className='flex grow items-center overflow-hidden'>

6
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
Unescape Escape View File

@ -6,7 +6,7 @@ import { useDebounceFn } from 'ahooks'
type FileListProps = { type FileListProps = {
fileList: OnlineDriveFile[] fileList: OnlineDriveFile[]
selectedFileList: string[] selectedFileKeys: string[]
prefix: string[] prefix: string[]
keywords: string keywords: string
bucket: string bucket: string
@ -27,7 +27,7 @@ type FileListProps = {
const FileList = ({ const FileList = ({
fileList, fileList,
selectedFileList, selectedFileKeys,
prefix, prefix,
keywords, keywords,
bucket, bucket,
@ -75,7 +75,7 @@ const FileList = ({
/> />
<List <List
fileList={fileList} fileList={fileList}
selectedFileList={selectedFileList} selectedFileKeys={selectedFileKeys}
keywords={keywords} keywords={keywords}
handleResetKeywords={handleResetKeywords} handleResetKeywords={handleResetKeywords}
handleOpenFolder={handleOpenFolder} handleOpenFolder={handleOpenFolder}

6
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
Unescape Escape View File

@ -9,7 +9,7 @@ import { useDataSourceStore } from '../../../store'
type FileListProps = { type FileListProps = {
fileList: OnlineDriveFile[] fileList: OnlineDriveFile[]
selectedFileList: string[] selectedFileKeys: string[]
keywords: string keywords: string
isInPipeline: boolean isInPipeline: boolean
isLoading: boolean isLoading: boolean
@ -26,7 +26,7 @@ type FileListProps = {
const List = ({ const List = ({
fileList, fileList,
selectedFileList, selectedFileKeys,
keywords, keywords,
handleResetKeywords, handleResetKeywords,
handleSelectFile, handleSelectFile,
@ -82,7 +82,7 @@ const List = ({
<div className='flex h-full flex-col gap-y-px overflow-y-auto rounded-[10px] bg-background-section px-1 py-1.5'> <div className='flex h-full flex-col gap-y-px overflow-y-auto rounded-[10px] bg-background-section px-1 py-1.5'>
{ {
fileList.map((file) => { fileList.map((file) => {
const isSelected = selectedFileList.includes(file.key) const isSelected = selectedFileKeys.includes(file.key)
return ( return (
<Item <Item
key={file.key} key={file.key}

32
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
Unescape Escape View File

@ -27,7 +27,7 @@ const OnlineDrive = ({
const prefix = useDataSourceStoreWithSelector(state => state.prefix) const prefix = useDataSourceStoreWithSelector(state => state.prefix)
const keywords = useDataSourceStoreWithSelector(state => state.keywords) const keywords = useDataSourceStoreWithSelector(state => state.keywords)
const bucket = useDataSourceStoreWithSelector(state => state.bucket) const bucket = useDataSourceStoreWithSelector(state => state.bucket)
const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys)
const fileList = useDataSourceStoreWithSelector(state => state.fileList) const fileList = useDataSourceStoreWithSelector(state => state.fileList)
const currentNodeIdRef = useDataSourceStoreWithSelector(state => state.currentNodeIdRef) const currentNodeIdRef = useDataSourceStoreWithSelector(state => state.currentNodeIdRef)
const dataSourceStore = useDataSourceStore() const dataSourceStore = useDataSourceStore()
@ -43,7 +43,7 @@ const OnlineDrive = ({
startAfter?: string startAfter?: string
fileList?: OnlineDriveFile[] fileList?: OnlineDriveFile[]
}) => { }) => {
const { startAfter } = dataSourceStore.getState() const { startAfter, prefix, bucket, fileList } = dataSourceStore.getState()
const _prefix = params.prefix ?? prefix const _prefix = params.prefix ?? prefix
const _bucket = params.bucket ?? bucket const _bucket = params.bucket ?? bucket
const _startAfter = params.startAfter ?? startAfter.current const _startAfter = params.startAfter ?? startAfter.current
@ -80,16 +80,22 @@ const OnlineDrive = ({
}, },
}, },
) )
}, [prefix, bucket, datasourceNodeRunURL, dataSourceStore, fileList]) }, [datasourceNodeRunURL, dataSourceStore])
useEffect(() => { useEffect(() => {
if (nodeId !== currentNodeIdRef.current) { if (nodeId !== currentNodeIdRef.current) {
const { setFileList, setBucket, setPrefix, setKeywords, setSelectedFileList } = dataSourceStore.getState() const {
setFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileKeys,
} = dataSourceStore.getState()
setFileList([]) setFileList([])
setBucket('') setBucket('')
setPrefix([]) setPrefix([])
setKeywords('') setKeywords('')
setSelectedFileList([]) setSelectedFileKeys([])
currentNodeIdRef.current = nodeId currentNodeIdRef.current = nodeId
getOnlineDriveFiles({ getOnlineDriveFiles({
prefix: [], prefix: [],
@ -124,9 +130,9 @@ const OnlineDrive = ({
}, [dataSourceStore]) }, [dataSourceStore])
const handleSelectFile = useCallback((file: OnlineDriveFile) => { const handleSelectFile = useCallback((file: OnlineDriveFile) => {
const { setSelectedFileList } = dataSourceStore.getState() const { selectedFileKeys, setSelectedFileKeys } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.bucket) return if (file.type === OnlineDriveFileType.bucket) return
const newSelectedFileList = produce(selectedFileList, (draft) => { const newSelectedFileList = produce(selectedFileKeys, (draft) => {
if (draft.includes(file.key)) { if (draft.includes(file.key)) {
const index = draft.indexOf(file.key) const index = draft.indexOf(file.key)
draft.splice(index, 1) draft.splice(index, 1)
@ -136,11 +142,11 @@ const OnlineDrive = ({
draft.push(file.key) draft.push(file.key)
} }
}) })
setSelectedFileList(newSelectedFileList) setSelectedFileKeys(newSelectedFileList)
}, [dataSourceStore, isInPipeline, selectedFileList]) }, [dataSourceStore, isInPipeline])
const handleOpenFolder = useCallback((file: OnlineDriveFile) => { const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
const { setPrefix, setBucket, setFileList, setSelectedFileList } = dataSourceStore.getState() const { prefix, setPrefix, setBucket, setFileList, setSelectedFileKeys } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.file) return if (file.type === OnlineDriveFileType.file) return
setFileList([]) setFileList([])
if (file.type === OnlineDriveFileType.bucket) { if (file.type === OnlineDriveFileType.bucket) {
@ -148,7 +154,7 @@ const OnlineDrive = ({
getOnlineDriveFiles({ bucket: file.displayName, fileList: [] }) getOnlineDriveFiles({ bucket: file.displayName, fileList: [] })
} }
else { else {
setSelectedFileList([]) setSelectedFileKeys([])
const displayName = file.displayName.endsWith('/') ? file.displayName.slice(0, -1) : file.displayName const displayName = file.displayName.endsWith('/') ? file.displayName.slice(0, -1) : file.displayName
const newPrefix = produce(prefix, (draft) => { const newPrefix = produce(prefix, (draft) => {
draft.push(displayName) draft.push(displayName)
@ -156,7 +162,7 @@ const OnlineDrive = ({
setPrefix(newPrefix) setPrefix(newPrefix)
getOnlineDriveFiles({ prefix: newPrefix, fileList: [] }) getOnlineDriveFiles({ prefix: newPrefix, fileList: [] })
} }
}, [dataSourceStore, getOnlineDriveFiles, prefix]) }, [dataSourceStore, getOnlineDriveFiles])
return ( return (
<div className='flex flex-col gap-y-2'> <div className='flex flex-col gap-y-2'>
@ -166,7 +172,7 @@ const OnlineDrive = ({
/> />
<FileList <FileList
fileList={onlineDriveFileList} fileList={onlineDriveFileList}
selectedFileList={selectedFileList} selectedFileKeys={selectedFileKeys}
prefix={prefix} prefix={prefix}
keywords={keywords} keywords={keywords}
bucket={bucket} bucket={bucket}

10
web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts
Unescape Escape View File

@ -17,7 +17,7 @@ export type OnlineDocumentSliceShape = {
previewOnlineDocumentRef: React.MutableRefObject<NotionPage | undefined> previewOnlineDocumentRef: React.MutableRefObject<NotionPage | undefined>
} }
export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set) => { export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set, get) => {
return ({ return ({
documentsData: [], documentsData: [],
setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({ setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({
@ -32,9 +32,13 @@ export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> =
currentWorkspaceId: workspaceId, currentWorkspaceId: workspaceId,
})), })),
onlineDocuments: [], onlineDocuments: [],
setOnlineDocuments: (documents: NotionPage[]) => set(() => ({ setOnlineDocuments: (documents: NotionPage[]) => {
set(() => ({
onlineDocuments: documents, onlineDocuments: documents,
})), }))
const { previewOnlineDocumentRef } = get()
previewOnlineDocumentRef.current = documents[0]
},
currentDocument: undefined, currentDocument: undefined,
setCurrentDocument: (document: NotionPage | undefined) => set(() => ({ setCurrentDocument: (document: NotionPage | undefined) => set(() => ({
currentDocument: document, currentDocument: document,

23
web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
Unescape Escape View File

@ -6,17 +6,18 @@ export type OnlineDriveSliceShape = {
setPrefix: (prefix: string[]) => void setPrefix: (prefix: string[]) => void
keywords: string keywords: string
setKeywords: (keywords: string) => void setKeywords: (keywords: string) => void
selectedFileList: string[] selectedFileKeys: string[]
setSelectedFileList: (selectedFileList: string[]) => void setSelectedFileKeys: (selectedFileKeys: string[]) => void
fileList: OnlineDriveFile[] fileList: OnlineDriveFile[]
setFileList: (fileList: OnlineDriveFile[]) => void setFileList: (fileList: OnlineDriveFile[]) => void
bucket: string bucket: string
setBucket: (bucket: string) => void setBucket: (bucket: string) => void
startAfter: React.MutableRefObject<string> startAfter: React.MutableRefObject<string>
isTruncated: React.MutableRefObject<boolean> isTruncated: React.MutableRefObject<boolean>
previewOnlineDriveFileRef: React.MutableRefObject<OnlineDriveFile | undefined>
} }
export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set) => { export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set, get) => {
return ({ return ({
prefix: [], prefix: [],
setPrefix: (prefix: string[]) => set(() => ({ setPrefix: (prefix: string[]) => set(() => ({
@ -26,11 +27,15 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
setKeywords: (keywords: string) => set(() => ({ setKeywords: (keywords: string) => set(() => ({
keywords, keywords,
})), })),
startAfter: { current: '' }, selectedFileKeys: [],
selectedFileList: [], setSelectedFileKeys: (selectedFileKeys: string[]) => {
setSelectedFileList: (selectedFileList: string[]) => set(() => ({ set(() => ({
selectedFileList, selectedFileKeys,
})), }))
const key = selectedFileKeys[0]
const { fileList, previewOnlineDriveFileRef } = get()
previewOnlineDriveFileRef.current = fileList.find(file => file.key === key)
},
fileList: [], fileList: [],
setFileList: (fileList: OnlineDriveFile[]) => set(() => ({ setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
fileList, fileList,
@ -39,6 +44,8 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
setBucket: (bucket: string) => set(() => ({ setBucket: (bucket: string) => set(() => ({
bucket, bucket,
})), })),
startAfter: { current: '' },
isTruncated: { current: false }, isTruncated: { current: false },
previewOnlineDriveFileRef: { current: undefined },
}) })
} }

10
web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts
Unescape Escape View File

@ -16,12 +16,16 @@ export type WebsiteCrawlSliceShape = {
previewWebsitePageRef: React.MutableRefObject<CrawlResultItem | undefined> previewWebsitePageRef: React.MutableRefObject<CrawlResultItem | undefined>
} }
export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set) => { export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set, get) => {
return ({ return ({
websitePages: [], websitePages: [],
setWebsitePages: (pages: CrawlResultItem[]) => set(() => ({ setWebsitePages: (pages: CrawlResultItem[]) => {
set(() => ({
websitePages: pages, websitePages: pages,
})), }))
const { previewWebsitePageRef } = get()
previewWebsitePageRef.current = pages[0]
},
currentWebsite: undefined, currentWebsite: undefined,
setCurrentWebsite: (website: CrawlResultItem | undefined) => set(() => ({ setCurrentWebsite: (website: CrawlResultItem | undefined) => set(() => ({
currentWebsite: website, currentWebsite: website,

2
web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx
Unescape Escape View File

@ -67,7 +67,6 @@ const WebsiteCrawl = ({
setPreviewIndex, setPreviewIndex,
setCurrentWebsite, setCurrentWebsite,
currentNodeIdRef, currentNodeIdRef,
previewWebsitePageRef,
} = dataSourceStore.getState() } = dataSourceStore.getState()
if (nodeId !== currentNodeIdRef.current) { if (nodeId !== currentNodeIdRef.current) {
setStep(CrawlStep.init) setStep(CrawlStep.init)
@ -75,7 +74,6 @@ const WebsiteCrawl = ({
setCurrentWebsite(undefined) setCurrentWebsite(undefined)
setWebsitePages([]) setWebsitePages([])
setPreviewIndex(0) setPreviewIndex(0)
previewWebsitePageRef.current = undefined
setCrawledNum(0) setCrawledNum(0)
setTotalNum(0) setTotalNum(0)
setCrawlErrorMessage('') setCrawlErrorMessage('')

15
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
Unescape Escape View File

@ -63,7 +63,6 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
export const useLocalFile = () => { export const useLocalFile = () => {
const fileList = useDataSourceStoreWithSelector(state => state.localFileList) const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
const previewFileRef = useDataSourceStoreWithSelector(state => state.previewLocalFileRef)
const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile) const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile)
const dataSourceStore = useDataSourceStore() const dataSourceStore = useDataSourceStore()
@ -76,7 +75,6 @@ export const useLocalFile = () => {
return { return {
fileList, fileList,
previewFileRef,
allFileLoaded, allFileLoaded,
currentLocalFile, currentLocalFile,
hidePreviewLocalFile, hidePreviewLocalFile,
@ -87,7 +85,6 @@ export const useOnlineDocuments = () => {
const documentsData = useDataSourceStoreWithSelector(state => state.documentsData) const documentsData = useDataSourceStoreWithSelector(state => state.documentsData)
const currentWorkspaceId = useDataSourceStoreWithSelector(state => state.currentWorkspaceId) const currentWorkspaceId = useDataSourceStoreWithSelector(state => state.currentWorkspaceId)
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments) const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
const previewOnlineDocumentRef = useDataSourceStoreWithSelector(state => state.previewOnlineDocumentRef)
const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument) const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument)
const dataSourceStore = useDataSourceStore() const dataSourceStore = useDataSourceStore()
@ -117,7 +114,6 @@ export const useOnlineDocuments = () => {
onlineDocuments, onlineDocuments,
currentDocument, currentDocument,
PagesMapAndSelectedPagesId, PagesMapAndSelectedPagesId,
previewOnlineDocumentRef,
hidePreviewOnlineDocument, hidePreviewOnlineDocument,
} }
} }
@ -125,7 +121,6 @@ export const useOnlineDocuments = () => {
export const useWebsiteCrawl = () => { export const useWebsiteCrawl = () => {
const websitePages = useDataSourceStoreWithSelector(state => state.websitePages) const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
const currentWebsite = useDataSourceStoreWithSelector(state => state.currentWebsite) const currentWebsite = useDataSourceStoreWithSelector(state => state.currentWebsite)
const previewWebsitePageRef = useDataSourceStoreWithSelector(state => state.previewWebsitePageRef)
const dataSourceStore = useDataSourceStore() const dataSourceStore = useDataSourceStore()
const hideWebsitePreview = useCallback(() => { const hideWebsitePreview = useCallback(() => {
@ -136,7 +131,6 @@ export const useWebsiteCrawl = () => {
return { return {
websitePages, websitePages,
previewWebsitePageRef,
currentWebsite, currentWebsite,
hideWebsitePreview, hideWebsitePreview,
} }
@ -144,10 +138,15 @@ export const useWebsiteCrawl = () => {
export const useOnlineDrive = () => { export const useOnlineDrive = () => {
const fileList = useDataSourceStoreWithSelector(state => state.fileList) const fileList = useDataSourceStoreWithSelector(state => state.fileList)
const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys)
const selectedOnlineDriveFileList = useMemo(() => {
return selectedFileKeys.map(key => fileList.find(item => item.key === key)!)
}, [fileList, selectedFileKeys])
return { return {
fileList, fileList,
selectedFileList, selectedFileKeys,
selectedOnlineDriveFileList,
} }
} }

69
web/app/components/datasets/documents/create-from-pipeline/index.tsx
Unescape Escape View File

@ -24,7 +24,7 @@ import WebsitePreview from './preview/web-preview'
import ProcessDocuments from './process-documents' import ProcessDocuments from './process-documents'
import ChunkPreview from './preview/chunk-preview' import ChunkPreview from './preview/chunk-preview'
import Processing from './processing' import Processing from './processing'
import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline' import type { InitialDocumentDetail, OnlineDriveFile, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline'
import { TransferMethod } from '@/types/app' import { TransferMethod } from '@/types/app'
import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks' import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
@ -57,7 +57,6 @@ const CreateFormPipeline = () => {
} = useAddDocumentsSteps() } = useAddDocumentsSteps()
const { const {
fileList, fileList,
previewFileRef,
allFileLoaded, allFileLoaded,
currentLocalFile, currentLocalFile,
hidePreviewLocalFile, hidePreviewLocalFile,
@ -67,18 +66,17 @@ const CreateFormPipeline = () => {
onlineDocuments, onlineDocuments,
currentDocument, currentDocument,
PagesMapAndSelectedPagesId, PagesMapAndSelectedPagesId,
previewOnlineDocumentRef,
hidePreviewOnlineDocument, hidePreviewOnlineDocument,
} = useOnlineDocuments() } = useOnlineDocuments()
const { const {
websitePages, websitePages,
previewWebsitePageRef,
currentWebsite, currentWebsite,
hideWebsitePreview, hideWebsitePreview,
} = useWebsiteCrawl() } = useWebsiteCrawl()
const { const {
fileList: onlineDriveFileList, fileList: onlineDriveFileList,
selectedFileList, selectedFileKeys,
selectedOnlineDriveFileList,
} = useOnlineDrive() } = useOnlineDrive()
const datasourceType = datasource?.nodeData.provider_type const datasourceType = datasource?.nodeData.provider_type
@ -107,9 +105,9 @@ const CreateFormPipeline = () => {
if (datasourceType === DatasourceType.websiteCrawl) if (datasourceType === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length return isShowVectorSpaceFull || !websitePages.length
if (datasourceType === DatasourceType.onlineDrive) if (datasourceType === DatasourceType.onlineDrive)
return isShowVectorSpaceFull || !selectedFileList.length return isShowVectorSpaceFull || !selectedFileKeys.length
return false return false
}, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileList.length]) }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileKeys.length])
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
file_size_limit: 15, file_size_limit: 15,
@ -143,8 +141,8 @@ const CreateFormPipeline = () => {
if (datasourceType === DatasourceType.onlineDocument) if (datasourceType === DatasourceType.onlineDocument)
return onlineDocuments.length return onlineDocuments.length
if (datasourceType === DatasourceType.onlineDrive) if (datasourceType === DatasourceType.onlineDrive)
return selectedFileList.length return selectedFileKeys.length
}, [datasourceType, onlineDocuments.length, selectedFileList.length]) }, [datasourceType, onlineDocuments.length, selectedFileKeys.length])
const tip = useMemo(() => { const tip = useMemo(() => {
if (datasourceType === DatasourceType.onlineDocument) if (datasourceType === DatasourceType.onlineDocument)
@ -163,9 +161,15 @@ const CreateFormPipeline = () => {
const handlePreviewChunks = useCallback(async (data: Record<string, any>) => { const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
if (!datasource) if (!datasource)
return return
const {
previewLocalFileRef,
previewOnlineDocumentRef,
previewWebsitePageRef,
previewOnlineDriveFileRef,
} = dataSourceStore.getState()
const datasourceInfoList: Record<string, any>[] = [] const datasourceInfoList: Record<string, any>[] = []
if (datasourceType === DatasourceType.localFile) { if (datasourceType === DatasourceType.localFile) {
const { id, name, type, size, extension, mime_type } = previewFileRef.current as File const { id, name, type, size, extension, mime_type } = previewLocalFileRef.current as File
const documentInfo = { const documentInfo = {
related_id: id, related_id: id,
name, name,
@ -188,6 +192,14 @@ const CreateFormPipeline = () => {
} }
if (datasourceType === DatasourceType.websiteCrawl) if (datasourceType === DatasourceType.websiteCrawl)
datasourceInfoList.push(previewWebsitePageRef.current!) datasourceInfoList.push(previewWebsitePageRef.current!)
if (datasourceType === DatasourceType.onlineDrive) {
const { bucket } = dataSourceStore.getState()
const { key } = previewOnlineDriveFileRef.current!
datasourceInfoList.push({
bucket,
key,
})
}
await runPublishedPipeline({ await runPublishedPipeline({
pipeline_id: pipelineId!, pipeline_id: pipelineId!,
inputs: data, inputs: data,
@ -200,7 +212,7 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
}, },
}) })
}, [datasource, datasourceType, previewWebsitePageRef, runPublishedPipeline, pipelineId, previewFileRef, previewOnlineDocumentRef]) }, [datasource, datasourceType, runPublishedPipeline, pipelineId, dataSourceStore])
const handleProcess = useCallback(async (data: Record<string, any>) => { const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource) if (!datasource)
@ -240,7 +252,7 @@ const CreateFormPipeline = () => {
if (datasourceType === DatasourceType.onlineDrive) { if (datasourceType === DatasourceType.onlineDrive) {
if (datasourceType === DatasourceType.onlineDrive) { if (datasourceType === DatasourceType.onlineDrive) {
const { bucket } = dataSourceStore.getState() const { bucket } = dataSourceStore.getState()
selectedFileList.forEach((key) => { selectedFileKeys.forEach((key) => {
datasourceInfoList.push({ datasourceInfoList.push({
bucket, bucket,
key, key,
@ -262,7 +274,7 @@ const CreateFormPipeline = () => {
handleNextStep() handleNextStep()
}, },
}) })
}, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileList, websitePages]) }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileKeys, websitePages])
const onClickProcess = useCallback(() => { const onClickProcess = useCallback(() => {
isPreview.current = false isPreview.current = false
@ -279,22 +291,31 @@ const CreateFormPipeline = () => {
}, [handlePreviewChunks, handleProcess]) }, [handlePreviewChunks, handleProcess])
const handlePreviewFileChange = useCallback((file: DocumentItem) => { const handlePreviewFileChange = useCallback((file: DocumentItem) => {
previewFileRef.current = file const { previewLocalFileRef } = dataSourceStore.getState()
previewLocalFileRef.current = file
onClickPreview() onClickPreview()
}, [onClickPreview, previewFileRef]) }, [dataSourceStore, onClickPreview])
const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
const { previewOnlineDocumentRef } = dataSourceStore.getState()
previewOnlineDocumentRef.current = page previewOnlineDocumentRef.current = page
onClickPreview() onClickPreview()
}, [onClickPreview, previewOnlineDocumentRef]) }, [dataSourceStore, onClickPreview])
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
const { previewWebsitePageRef } = dataSourceStore.getState()
previewWebsitePageRef.current = website previewWebsitePageRef.current = website
onClickPreview() onClickPreview()
}, [onClickPreview, previewWebsitePageRef]) }, [dataSourceStore, onClickPreview])
const handlePreviewOnlineDriveFileChange = useCallback((file: OnlineDriveFile) => {
const { previewOnlineDriveFileRef } = dataSourceStore.getState()
previewOnlineDriveFileRef.current = file
onClickPreview()
}, [dataSourceStore, onClickPreview])
const handleSelectAll = useCallback(() => { const handleSelectAll = useCallback(() => {
const { setOnlineDocuments, setSelectedFileList, setSelectedPagesId } = dataSourceStore.getState() const { setOnlineDocuments, setSelectedFileKeys, setSelectedPagesId } = dataSourceStore.getState()
if (datasourceType === DatasourceType.onlineDocument) { if (datasourceType === DatasourceType.onlineDocument) {
const allIds = currentWorkspace?.pages.map(page => page.page_id) || [] const allIds = currentWorkspace?.pages.map(page => page.page_id) || []
if (onlineDocuments.length < allIds.length) { if (onlineDocuments.length < allIds.length) {
@ -311,12 +332,12 @@ const CreateFormPipeline = () => {
const allKeys = onlineDriveFileList.filter((item) => { const allKeys = onlineDriveFileList.filter((item) => {
return item.type !== 'bucket' return item.type !== 'bucket'
}).map(file => file.key) }).map(file => file.key)
if (selectedFileList.length < allKeys.length) if (selectedFileKeys.length < allKeys.length)
setSelectedFileList(allKeys) setSelectedFileKeys(allKeys)
else else
setSelectedFileList([]) setSelectedFileKeys([])
} }
}, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType, onlineDocuments.length, onlineDriveFileList, selectedFileList.length]) }, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType, onlineDocuments.length, onlineDriveFileList, selectedFileKeys.length])
if (isFetchingPipelineInfo) { if (isFetchingPipelineInfo) {
return ( return (
@ -440,9 +461,10 @@ const CreateFormPipeline = () => {
<div className='flex h-full flex-col pl-2 pt-2'> <div className='flex h-full flex-col pl-2 pt-2'>
<ChunkPreview <ChunkPreview
dataSourceType={datasourceType as DatasourceType} dataSourceType={datasourceType as DatasourceType}
files={fileList.map(file => file.file)} localFiles={fileList.map(file => file.file)}
onlineDocuments={onlineDocuments} onlineDocuments={onlineDocuments}
websitePages={websitePages} websitePages={websitePages}
onlineDriveFiles={selectedOnlineDriveFileList}
isIdle={isIdle} isIdle={isIdle}
isPending={isPending && isPreview.current} isPending={isPending && isPreview.current}
estimateData={estimateData} estimateData={estimateData}
@ -450,6 +472,7 @@ const CreateFormPipeline = () => {
handlePreviewFileChange={handlePreviewFileChange} handlePreviewFileChange={handlePreviewFileChange}
handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange} handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
handlePreviewWebsitePageChange={handlePreviewWebsiteChange} handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange}
/> />
</div> </div>
</div> </div>

37
web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx
Unescape Escape View File

@ -14,13 +14,15 @@ import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from
import { RiSearchEyeLine } from '@remixicon/react' import { RiSearchEyeLine } from '@remixicon/react'
import Badge from '@/app/components/base/badge' import Badge from '@/app/components/base/badge'
import Button from '@/app/components/base/button' import Button from '@/app/components/base/button'
import type { OnlineDriveFile } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline'
type ChunkPreviewProps = { type ChunkPreviewProps = {
dataSourceType: DatasourceType dataSourceType: DatasourceType
files: CustomFile[] localFiles: CustomFile[]
onlineDocuments: NotionPage[] onlineDocuments: NotionPage[]
websitePages: CrawlResultItem[] websitePages: CrawlResultItem[]
onlineDriveFiles: OnlineDriveFile[]
isIdle: boolean isIdle: boolean
isPending: boolean isPending: boolean
estimateData: FileIndexingEstimateResponse | undefined estimateData: FileIndexingEstimateResponse | undefined
@ -28,13 +30,15 @@ type ChunkPreviewProps = {
handlePreviewFileChange: (file: DocumentItem) => void handlePreviewFileChange: (file: DocumentItem) => void
handlePreviewOnlineDocumentChange: (page: NotionPage) => void handlePreviewOnlineDocumentChange: (page: NotionPage) => void
handlePreviewWebsitePageChange: (page: CrawlResultItem) => void handlePreviewWebsitePageChange: (page: CrawlResultItem) => void
handlePreviewOnlineDriveFileChange: (file: OnlineDriveFile) => void
} }
const ChunkPreview = ({ const ChunkPreview = ({
dataSourceType, dataSourceType,
files, localFiles,
onlineDocuments, onlineDocuments,
websitePages, websitePages,
onlineDriveFiles,
isIdle, isIdle,
isPending, isPending,
estimateData, estimateData,
@ -42,13 +46,15 @@ const ChunkPreview = ({
handlePreviewFileChange, handlePreviewFileChange,
handlePreviewOnlineDocumentChange, handlePreviewOnlineDocumentChange,
handlePreviewWebsitePageChange, handlePreviewWebsitePageChange,
handlePreviewOnlineDriveFileChange,
}: ChunkPreviewProps) => { }: ChunkPreviewProps) => {
const { t } = useTranslation() const { t } = useTranslation()
const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form) const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form)
const [previewFile, setPreviewFile] = useState<DocumentItem>(files[0] as DocumentItem) const [previewFile, setPreviewFile] = useState<DocumentItem>(localFiles[0] as DocumentItem)
const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0]) const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0])
const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0]) const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0])
const [previewOnlineDriveFile, setPreviewOnlineDriveFile] = useState<OnlineDriveFile>(onlineDriveFiles[0])
return ( return (
<PreviewContainer <PreviewContainer
@ -58,7 +64,7 @@ const ChunkPreview = ({
<div className='flex items-center gap-1'> <div className='flex items-center gap-1'>
{dataSourceType === DatasourceType.localFile {dataSourceType === DatasourceType.localFile
&& <PreviewDocumentPicker && <PreviewDocumentPicker
files={files as Array<Required<CustomFile>>} files={localFiles as Array<Required<CustomFile>>}
onChange={(selected) => { onChange={(selected) => {
setPreviewFile(selected) setPreviewFile(selected)
handlePreviewFileChange(selected) handlePreviewFileChange(selected)
@ -110,6 +116,29 @@ const ChunkPreview = ({
} }
/> />
} }
{dataSourceType === DatasourceType.onlineDrive
&& <PreviewDocumentPicker
files={
onlineDriveFiles.map(file => ({
id: file.key,
name: file.displayName,
extension: 'md',
}))
}
onChange={(selected) => {
const selectedFile = onlineDriveFiles.find(file => file.key === selected.id)
setPreviewOnlineDriveFile(selectedFile!)
handlePreviewOnlineDriveFileChange(selectedFile!)
}}
value={
{
id: previewOnlineDriveFile?.key || '',
name: previewOnlineDriveFile?.displayName || '',
extension: 'md',
}
}
/>
}
{ {
currentDocForm !== ChunkingMode.qa currentDocForm !== ChunkingMode.qa
&& <Badge text={t('datasetCreation.stepTwo.previewChunkCount', { && <Badge text={t('datasetCreation.stepTwo.previewChunkCount', {

10
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
Unescape Escape View File

@ -23,7 +23,7 @@ const TestRunPanel = () => {
const fileList = useDataSourceStoreWithSelector(state => state.localFileList) const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments) const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
const websitePages = useDataSourceStoreWithSelector(state => state.websitePages) const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys)
const dataSourceStore = useDataSourceStore() const dataSourceStore = useDataSourceStore()
const [datasource, setDatasource] = useState<Datasource>() const [datasource, setDatasource] = useState<Datasource>()
@ -45,9 +45,9 @@ const TestRunPanel = () => {
if (datasourceType === DatasourceType.websiteCrawl) if (datasourceType === DatasourceType.websiteCrawl)
return !websitePages.length return !websitePages.length
if (datasourceType === DatasourceType.onlineDrive) if (datasourceType === DatasourceType.onlineDrive)
return !selectedFileList.length return !selectedFileKeys.length
return false return false
}, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileList.length, websitePages.length]) }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileKeys.length, websitePages.length])
const handleClose = useCallback(() => { const handleClose = useCallback(() => {
setShowDebugAndPreviewPanel(false) setShowDebugAndPreviewPanel(false)
@ -87,7 +87,7 @@ const TestRunPanel = () => {
const { bucket } = dataSourceStore.getState() const { bucket } = dataSourceStore.getState()
datasourceInfoList.push({ datasourceInfoList.push({
bucket, bucket,
key: selectedFileList[0], key: selectedFileKeys[0],
}) })
} }
handleRun({ handleRun({
@ -96,7 +96,7 @@ const TestRunPanel = () => {
datasource_type: datasourceType, datasource_type: datasourceType,
datasource_info_list: datasourceInfoList, datasource_info_list: datasourceInfoList,
}) })
}, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileList, websitePages]) }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileKeys, websitePages])
return ( return (
<div <div

4
web/app/components/workflow/plugin-dependency/hooks.ts
Unescape Escape View File

@ -8,8 +8,8 @@ export const usePluginDependencies = () => {
const { mutateAsync: checkPipelineDependencies } = useCheckPipelineDependencies() const { mutateAsync: checkPipelineDependencies } = useCheckPipelineDependencies()
const handleCheckPluginDependencies = useCallback(async (id: string, isPipeline = false) => { const handleCheckPluginDependencies = useCallback(async (id: string, isPipeline = false) => {
const mutateAsync = isPipeline ? checkPipelineDependencies : checkWorkflowDependencies const checkDependencies = isPipeline ? checkPipelineDependencies : checkWorkflowDependencies
const { leaked_dependencies } = await mutateAsync(id) const { leaked_dependencies } = await checkDependencies(id)
const { setDependencies } = usePluginDependenciesStore.getState() const { setDependencies } = usePluginDependenciesStore.getState()
setDependencies(leaked_dependencies) setDependencies(leaked_dependencies)
}, [checkWorkflowDependencies, checkPipelineDependencies]) }, [checkWorkflowDependencies, checkPipelineDependencies])

Write Preview
Loading…
Cancel
Save