ngskcloud
/
gcgj-dify-1.7.0
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: refactor data source handling and integrate OnlineDrive component in TestRunPanel

Browse Source
wtw/rag-pipeline
twwu 9 months ago
parent 025b55ef3b
commit 3a9c79b09a
14 changed files with 199 additions and 128 deletions
Show Stats Download Patch File Download Diff File

12
web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx
Unescape Escape View File

@ -16,14 +16,18 @@ const DataSourceOptions = ({
datasourceNodeId,
onSelect,
}: DataSourceOptionsProps) => {
const { datasources, options } = useDatasourceOptions(pipelineNodes)
const options = useDatasourceOptions(pipelineNodes)
const handelSelect = useCallback((value: string) => {
const selectedOption = datasources.find(option => option.nodeId === value)
const selectedOption = options.find(option => option.value === value)
if (!selectedOption)
return
onSelect(selectedOption)
}, [datasources, onSelect])
const datasource = {
nodeId: selectedOption.value,
nodeData: selectedOption.data,
}
onSelect(datasource)
}, [onSelect, options])
useEffect(() => {
if (options.length > 0 && !datasourceNodeId)

36
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
Unescape Escape View File

@ -1,10 +1,9 @@
import { useTranslation } from 'react-i18next'
import { AddDocumentsStep } from './types'
import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import { useCallback, useMemo, useRef, useState } from 'react'
import { BlockEnum, type Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { DatasourceType } from '@/models/pipeline'
import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets'
import { CrawlStep } from '@/models/datasets'
import produce from 'immer'
@ -47,18 +46,6 @@ export const useAddDocumentsSteps = () => {
export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
const datasources: Datasource[] = useMemo(() => {
return datasourceNodes.map((node) => {
return {
nodeId: node.id,
type: node.data.provider_type as DatasourceType,
description: node.data.datasource_label,
docTitle: 'How to use?',
docLink: '',
fileExtensions: node.data.fileExtensions || [],
}
})
}, [datasourceNodes])
const options = useMemo(() => {
const options: DataSourceOption[] = []
@ -70,10 +57,29 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
data: node.data,
})
})
// todo: delete mock data
options.push({
label: 'Google Drive',
value: '123456',
// @ts-expect-error mock data
data: {
datasource_parameters: {},
datasource_configurations: {},
type: BlockEnum.DataSource,
title: 'Google Drive',
plugin_id: 'langgenius/google-drive',
provider_type: 'online_drive',
provider_name: 'google_drive',
datasource_name: 'google-drive',
datasource_label: 'Google Drive',
selected: false,
},
})
return options
}, [datasourceNodes])
return { datasources, options }
return options
}
export const useLocalFile = () => {

57
web/app/components/datasets/documents/create-from-pipeline/index.tsx
Unescape Escape View File

@ -84,17 +84,18 @@ const CreateFormPipeline = () => {
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
const datasourceType = datasource?.nodeData.provider_type
const nextBtnDisabled = useMemo(() => {
if (!datasource) return true
if (datasource.type === DatasourceType.localFile)
if (datasourceType === DatasourceType.localFile)
return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
if (datasource.type === DatasourceType.onlineDocument)
if (datasourceType === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasource.type === DatasourceType.websiteCrawl)
if (datasourceType === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length
return false
}, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
}, [datasource, datasourceType, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
@ -102,7 +103,7 @@ const CreateFormPipeline = () => {
if (!datasource)
return
const datasourceInfoList: Record<string, any>[] = []
if (datasource.type === DatasourceType.localFile) {
if (datasourceType === DatasourceType.localFile) {
const { id, name, type, size, extension, mime_type } = previewFile.current as File
const documentInfo = {
related_id: id,
@ -116,7 +117,7 @@ const CreateFormPipeline = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.onlineDocument) {
if (datasourceType === DatasourceType.onlineDocument) {
const { workspace_id, ...rest } = previewOnlineDocument.current
const documentInfo = {
workspace_id,
@ -124,13 +125,13 @@ const CreateFormPipeline = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl)
if (datasourceType === DatasourceType.websiteCrawl)
datasourceInfoList.push(previewWebsitePage.current)
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasource.type,
datasource_type: datasourceType as DatasourceType,
datasource_info_list: datasourceInfoList,
is_preview: true,
}, {
@ -138,13 +139,13 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
}, [datasource, datasourceType, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource)
return
const datasourceInfoList: Record<string, any>[] = []
if (datasource.type === DatasourceType.localFile) {
if (datasourceType === DatasourceType.localFile) {
fileList.forEach((file) => {
const { id, name, type, size, extension, mime_type } = file.file
const documentInfo = {
@ -160,7 +161,7 @@ const CreateFormPipeline = () => {
datasourceInfoList.push(documentInfo)
})
}
if (datasource.type === DatasourceType.onlineDocument) {
if (datasourceType === DatasourceType.onlineDocument) {
onlineDocuments.forEach((page) => {
const { workspace_id, ...rest } = page
const documentInfo = {
@ -170,7 +171,7 @@ const CreateFormPipeline = () => {
datasourceInfoList.push(documentInfo)
})
}
if (datasource.type === DatasourceType.websiteCrawl) {
if (datasourceType === DatasourceType.websiteCrawl) {
websitePages.forEach((websitePage) => {
datasourceInfoList.push(websitePage)
})
@ -179,7 +180,7 @@ const CreateFormPipeline = () => {
pipeline_id: pipelineId!,
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasource.type,
datasource_type: datasourceType as DatasourceType,
datasource_info_list: datasourceInfoList,
is_preview: false,
}, {
@ -189,7 +190,7 @@ const CreateFormPipeline = () => {
handleNextStep()
},
})
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
}, [datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
const onClickProcess = useCallback(() => {
isPreview.current = false
@ -246,38 +247,30 @@ const CreateFormPipeline = () => {
onSelect={setDatasource}
pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
/>
{datasource?.type === DatasourceType.localFile && (
{datasourceType === DatasourceType.localFile && (
<LocalFile
files={fileList}
allowedExtensions={datasource?.fileExtensions || []}
allowedExtensions={datasource!.nodeData.fileExtensions || []}
updateFile={updateFile}
updateFileList={updateFileList}
onPreview={updateCurrentFile}
notSupportBatchUpload={notSupportBatchUpload}
/>
)}
{datasource?.type === DatasourceType.onlineDocument && (
{datasourceType === DatasourceType.onlineDocument && (
<OnlineDocuments
nodeId={datasource?.nodeId || ''}
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
docLink: datasource.docLink || '',
}}
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onlineDocuments={onlineDocuments}
updateOnlineDocuments={updateOnlineDocuments}
canPreview
onPreview={updateCurrentPage}
/>
)}
{datasource?.type === DatasourceType.websiteCrawl && (
{datasourceType === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
docLink: datasource.docLink || '',
}}
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
crawlResult={crawlResult}
setCrawlResult={setCrawlResult}
step={step}
@ -299,7 +292,7 @@ const CreateFormPipeline = () => {
currentStep === 2 && (
<ProcessDocuments
ref={formRef}
dataSourceNodeId={datasource?.nodeId || ''}
dataSourceNodeId={datasource!.nodeId}
onProcess={onClickProcess}
onPreview={onClickPreview}
onSubmit={handleSubmit}
@ -341,7 +334,7 @@ const CreateFormPipeline = () => {
<div className='h-full min-w-0 flex-1'>
<div className='flex h-full flex-col pl-2 pt-2'>
<ChunkPreview
dataSourceType={datasource!.type}
dataSourceType={datasourceType as DatasourceType}
files={fileList.map(file => file.file)}
onlineDocuments={onlineDocuments}
websitePages={websitePages}

15
web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
Unescape Escape View File

@ -12,14 +12,18 @@ const DataSourceOptions = ({
dataSourceNodeId,
onSelect,
}: DataSourceOptionsProps) => {
const { datasources, options } = useDatasourceOptions()
const options = useDatasourceOptions()
const handelSelect = useCallback((value: string) => {
const selectedOption = datasources.find(option => option.nodeId === value)
const selectedOption = options.find(option => option.value === value)
if (!selectedOption)
return
onSelect(selectedOption)
}, [datasources, onSelect])
const datasource = {
nodeId: selectedOption.value,
nodeData: selectedOption.data,
}
onSelect(datasource)
}, [onSelect, options])
useEffect(() => {
if (options.length > 0 && !dataSourceNodeId)
@ -33,9 +37,10 @@ const DataSourceOptions = ({
<OptionCard
key={option.value}
label={option.label}
value={option.value}
nodeData={option.data}
selected={dataSourceNodeId === option.value}
onClick={handelSelect.bind(null, option.value)}
onClick={handelSelect}
/>
))}
</div>

12
web/app/components/rag-pipeline/components/panel/test-run/data-source-options/option-card.tsx
Unescape Escape View File

@ -1,4 +1,4 @@
import React from 'react'
import React, { useCallback } from 'react'
import cn from '@/utils/classnames'
import BlockIcon from '@/app/components/workflow/block-icon'
import { BlockEnum } from '@/app/components/workflow/types'
@ -7,19 +7,25 @@ import { useToolIcon } from '@/app/components/workflow/hooks'
type OptionCardProps = {
label: string
value: string
selected: boolean
nodeData: DataSourceNodeType
onClick?: () => void
onClick?: (value: string) => void
}
const OptionCard = ({
label,
value,
selected,
nodeData,
onClick,
}: OptionCardProps) => {
const toolIcon = useToolIcon(nodeData)
const handleClickCard = useCallback(() => {
onClick?.(value)
}, [value, onClick])
return (
<div
className={cn(
@ -28,7 +34,7 @@ const OptionCard = ({
? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg shadow-xs ring-[0.5px] ring-inset ring-components-option-card-option-selected-border'
: 'hover:bg-components-option-card-bg-hover hover:border-components-option-card-option-border-hover hover:shadow-xs',
)}
onClick={onClick}
onClick={handleClickCard}
>
<div className='flex size-7 items-center justify-center rounded-lg border-[0.5px] border-components-panel-border bg-background-default-dodge p-1'>
<BlockIcon

11
web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx
Unescape Escape View File

@ -1,13 +1,10 @@
import type { NotionPage } from '@/models/common'
import OnlineDocumentSelector from './online-document-selector'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
type OnlineDocumentsProps = {
nodeId: string
headerInfo: {
title: string
docTitle: string
docLink: string
}
nodeData: DataSourceNodeType
onlineDocuments: NotionPage[]
updateOnlineDocuments: (value: NotionPage[]) => void
canPreview?: boolean
@ -17,7 +14,7 @@ type OnlineDocumentsProps = {
const OnlineDocuments = ({
nodeId,
headerInfo,
nodeData,
onlineDocuments,
updateOnlineDocuments,
canPreview = false,
@ -27,7 +24,7 @@ const OnlineDocuments = ({
return (
<OnlineDocumentSelector
nodeId={nodeId}
headerInfo={headerInfo}
nodeData={nodeData}
value={onlineDocuments.map(page => page.page_id)}
onSelect={updateOnlineDocuments}
canPreview={canPreview}

17
web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx
Unescape Escape View File

@ -9,6 +9,7 @@ import { DatasourceType } from '@/models/pipeline'
import { ssePost } from '@/service/base'
import Toast from '@/app/components/base/toast'
import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
type OnlineDocumentSelectorProps = {
value?: string[]
@ -18,11 +19,7 @@ type OnlineDocumentSelectorProps = {
onPreview?: (selectedPage: NotionPage) => void
isInPipeline?: boolean
nodeId: string
headerInfo: {
title: string
docTitle: string
docLink: string
}
nodeData: DataSourceNodeType
}
const OnlineDocumentSelector = ({
@ -33,7 +30,7 @@ const OnlineDocumentSelector = ({
onPreview,
isInPipeline = false,
nodeId,
headerInfo,
nodeData,
}: OnlineDocumentSelectorProps) => {
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
@ -118,6 +115,14 @@ const OnlineDocumentSelector = ({
setCurrentWorkspaceId(firstWorkspaceId)
}, [firstWorkspaceId])
const headerInfo = useMemo(() => {
return {
title: nodeData.title,
docTitle: 'How to use?',
docLink: 'https://docs.dify.ai',
}
}, [nodeData])
if (!documentsData?.length)
return null

34
web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/connect/index.tsx
Unescape Escape View File

@ -0,0 +1,34 @@
import BlockIcon from '@/app/components/workflow/block-icon'
import { useToolIcon } from '@/app/components/workflow/hooks'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { BlockEnum } from '@/app/components/workflow/types'
type ConnectProps = {
nodeData: DataSourceNodeType
}
const Connect = ({
nodeData,
}: ConnectProps) => {
const toolIcon = useToolIcon(nodeData)
return (
<div className='flex flex-col p-6'>
<div className='flex size-12 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg p-1 shadow-lg shadow-shadow-shadow-5'>
<BlockIcon
type={BlockEnum.DataSource}
toolIcon={toolIcon}
size='md'
/>
</div>
<p className='mb-6 text-gray-600'>
To connect your online drive, please follow the instructions provided by your service provider.
</p>
<button className='rounded bg-blue-500 px-4 py-2 text-white hover:bg-blue-600'>
Connect Now
</button>
</div>
)
}
export default Connect

16
web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx
Unescape Escape View File

@ -0,0 +1,16 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import Connect from './connect'
type OnlineDriveProps = {
nodeData: DataSourceNodeType
}
const OnlineDrive = ({
nodeData,
}: OnlineDriveProps) => {
return (
<Connect nodeData={nodeData} />
)
}
export default OnlineDrive

19
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
Unescape Escape View File

@ -1,5 +1,5 @@
'use client'
import React, { useCallback, useEffect, useRef, useState } from 'react'
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
import { CrawlStep } from '@/models/datasets'
@ -19,22 +19,19 @@ import type {
DataSourceNodeCompletedResponse,
DataSourceNodeProcessingResponse,
} from '@/types/pipeline'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
export type CrawlerProps = {
nodeId: string
nodeData: DataSourceNodeType
crawlResult: CrawlResult | undefined
setCrawlResult: (payload: CrawlResult) => void
step: CrawlStep
setStep: (step: CrawlStep) => void
checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
headerInfo: {
title: string
docTitle: string
docLink: string
}
previewIndex?: number
onPreview?: (payload: CrawlResultItem, index: number) => void
isInPipeline?: boolean
@ -42,12 +39,12 @@ export type CrawlerProps = {
const Crawler = ({
nodeId,
nodeData,
crawlResult,
setCrawlResult,
step,
setStep,
checkedCrawlResult,
headerInfo,
onCheckedCrawlResultChange,
previewIndex,
onPreview,
@ -125,6 +122,14 @@ const Crawler = ({
handleRun(value)
}, [handleRun])
const headerInfo = useMemo(() => {
return {
title: nodeData.title,
docTitle: 'How to use?',
docLink: 'https://docs.dify.ai',
}
}, [nodeData])
return (
<div className='flex flex-col'>
<Header

4
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx
Unescape Escape View File

@ -7,12 +7,12 @@ type WebsiteCrawlProps = CrawlerProps
const WebsiteCrawl = ({
nodeId,
nodeData,
crawlResult,
setCrawlResult,
step,
setStep,
checkedCrawlResult,
headerInfo,
onCheckedCrawlResultChange,
previewIndex,
onPreview,
@ -21,12 +21,12 @@ const WebsiteCrawl = ({
return (
<Crawler
nodeId={nodeId}
nodeData={nodeData}
crawlResult={crawlResult}
setCrawlResult={setCrawlResult}
step={step}
setStep={setStep}
checkedCrawlResult={checkedCrawlResult}
headerInfo={headerInfo}
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
previewIndex={previewIndex}
onPreview={onPreview}

35
web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
Unescape Escape View File

@ -1,11 +1,10 @@
import { useTranslation } from 'react-i18next'
import type { DataSourceOption, Datasource } from './types'
import type { DataSourceOption } from './types'
import { TestRunStep } from './types'
import { useNodes } from 'reactflow'
import { BlockEnum } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { useCallback, useMemo, useState } from 'react'
import type { DatasourceType } from '@/models/pipeline'
import type { CrawlResult } from '@/models/datasets'
import { type CrawlResultItem, CrawlStep, type FileItem } from '@/models/datasets'
import produce from 'immer'
@ -45,18 +44,6 @@ export const useTestRunSteps = () => {
export const useDatasourceOptions = () => {
const nodes = useNodes<DataSourceNodeType>()
const datasourceNodes = nodes.filter(node => node.data.type === BlockEnum.DataSource)
const datasources: Datasource[] = useMemo(() => {
return datasourceNodes.map((node) => {
return {
nodeId: node.id,
type: node.data.provider_type as DatasourceType,
description: node.data.datasource_label,
docTitle: 'How to use?',
docLink: '',
fileExtensions: node.data.fileExtensions || [],
}
})
}, [datasourceNodes])
const options = useMemo(() => {
const options: DataSourceOption[] = []
@ -68,10 +55,28 @@ export const useDatasourceOptions = () => {
data: node.data,
})
})
// todo: delete mock data
options.push({
label: 'Google Drive',
value: '123456',
// @ts-expect-error mock data
data: {
datasource_parameters: {},
datasource_configurations: {},
type: BlockEnum.DataSource,
title: 'Google Drive',
plugin_id: 'langgenius/google-drive',
provider_type: 'online_drive',
provider_name: 'google_drive',
datasource_name: 'google-drive',
datasource_label: 'Google Drive',
selected: false,
},
})
return options
}, [datasourceNodes])
return { datasources, options }
return options
}
export const useLocalFile = () => {

52
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
Unescape Escape View File

@ -16,6 +16,7 @@ import { TransferMethod } from '@/types/app'
import CloseButton from './close-button'
import Header from './header'
import FooterTips from './footer-tips'
import OnlineDrive from './data-source/online-drive'
const TestRunPanel = () => {
const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
@ -51,17 +52,18 @@ const TestRunPanel = () => {
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
const datasourceType = datasource?.nodeData.provider_type
const nextBtnDisabled = useMemo(() => {
if (!datasource) return true
if (datasource.type === DatasourceType.localFile)
if (datasourceType === DatasourceType.localFile)
return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
if (datasource.type === DatasourceType.onlineDocument)
if (datasourceType === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasource.type === DatasourceType.websiteCrawl)
if (datasourceType === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length
return false
}, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
}, [datasource, datasourceType, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
const handleClose = () => {
setShowDebugAndPreviewPanel(false)
@ -71,7 +73,7 @@ const TestRunPanel = () => {
if (!datasource)
return
const datasourceInfoList: Record<string, any>[] = []
if (datasource.type === DatasourceType.localFile) {
if (datasourceType === DatasourceType.localFile) {
const { id, name, type, size, extension, mime_type } = fileList[0].file
const documentInfo = {
related_id: id,
@ -85,7 +87,7 @@ const TestRunPanel = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.onlineDocument) {
if (datasourceType === DatasourceType.onlineDocument) {
const { workspace_id, ...rest } = onlineDocuments[0]
const documentInfo = {
workspace_id,
@ -93,15 +95,15 @@ const TestRunPanel = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl)
if (datasourceType === DatasourceType.websiteCrawl)
datasourceInfoList.push(websitePages[0])
handleRun({
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasource.type,
datasource_type: datasourceType,
datasource_info_list: datasourceInfoList,
})
}, [datasource, fileList, handleRun, onlineDocuments, websitePages])
}, [datasource, datasourceType, fileList, handleRun, onlineDocuments, websitePages])
return (
<div
@ -118,37 +120,29 @@ const TestRunPanel = () => {
dataSourceNodeId={datasource?.nodeId || ''}
onSelect={setDatasource}
/>
{datasource?.type === DatasourceType.localFile && (
{datasourceType === DatasourceType.localFile && (
<LocalFile
files={fileList}
allowedExtensions={datasource?.fileExtensions || []}
allowedExtensions={datasource!.nodeData.fileExtensions || []}
updateFile={updateFile}
updateFileList={updateFileList}
notSupportBatchUpload={false} // only support single file upload in test run
/>
)}
{datasource?.type === DatasourceType.onlineDocument && (
{datasourceType === DatasourceType.onlineDocument && (
<OnlineDocuments
nodeId={datasource?.nodeId || ''}
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
docLink: datasource.docLink || '',
}}
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
onlineDocuments={onlineDocuments}
updateOnlineDocuments={updateOnlineDocuments}
isInPipeline
/>
)}
{datasource?.type === DatasourceType.websiteCrawl && (
{datasourceType === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
nodeId={datasource!.nodeId}
checkedCrawlResult={websitePages}
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
docLink: datasource.docLink || '',
}}
nodeData={datasource!.nodeData}
crawlResult={crawlResult}
setCrawlResult={setCrawlResult}
step={step}
@ -157,6 +151,12 @@ const TestRunPanel = () => {
isInPipeline
/>
)}
{datasourceType === DatasourceType.onlineDrive && (
<OnlineDrive
nodeData={datasource!.nodeData}
/>
)
}
{isShowVectorSpaceFull && (
<VectorSpaceFull />
)}
@ -169,7 +169,7 @@ const TestRunPanel = () => {
{
currentStep === 2 && (
<DocumentProcessing
dataSourceNodeId={datasource?.nodeId || ''}
dataSourceNodeId={datasource!.nodeId}
onProcess={handleProcess}
onBack={handleBackStep}
/>

7
web/app/components/rag-pipeline/components/panel/test-run/types.ts
Unescape Escape View File

@ -1,5 +1,4 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { DatasourceType } from '@/models/pipeline'
export enum TestRunStep {
dataSource = 'dataSource',
@ -14,9 +13,5 @@ export type DataSourceOption = {
export type Datasource = {
nodeId: string
type: DatasourceType
description: string
docTitle?: string
docLink?: string
fileExtensions?: string[]
nodeData: DataSourceNodeType
}

Write Preview
Loading…
Cancel
Save