ngskcloud
/
gcgj-dify-1.7.0
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: update website crawl handling and improve parameter naming in pipeline processing

Browse Source
pull/21398/head
twwu 11 months ago
parent 547bd3cc1b
commit 3e2f12b065
6 changed files with 25 additions and 32 deletions
Show Stats Download Patch File Download Diff File

7
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
Unescape Escape View File

@ -162,11 +162,16 @@ export const useWebsiteCrawl = () => {
setCurrentWebsite(undefined) setCurrentWebsite(undefined)
}, []) }, [])
const updataCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
setWebsitePages(checkedCrawlResult)
previewWebsitePage.current = checkedCrawlResult[0]
}, [])
return { return {
websitePages, websitePages,
websiteCrawlJobId, websiteCrawlJobId,
previewWebsitePage, previewWebsitePage,
setWebsitePages, updataCheckedCrawlResultChange,
setWebsiteCrawlJobId, setWebsiteCrawlJobId,
currentWebsite, currentWebsite,
updateCurrentWebsite, updateCurrentWebsite,

27
web/app/components/datasets/documents/create-from-pipeline/index.tsx
Unescape Escape View File

@ -72,9 +72,9 @@ const CreateFormPipeline = () => {
} = useOnlineDocuments() } = useOnlineDocuments()
const { const {
websitePages, websitePages,
websiteCrawlJobId, // websiteCrawlJobId, // todo: Add status query
previewWebsitePage, previewWebsitePage,
setWebsitePages, updataCheckedCrawlResultChange,
setWebsiteCrawlJobId, setWebsiteCrawlJobId,
currentWebsite, currentWebsite,
updateCurrentWebsite, updateCurrentWebsite,
@ -124,13 +124,8 @@ const CreateFormPipeline = () => {
} }
datasourceInfoList.push(documentInfo) datasourceInfoList.push(documentInfo)
} }
if (datasource.type === DatasourceType.websiteCrawl) { if (datasource.type === DatasourceType.websiteCrawl)
const documentInfo = { datasourceInfoList.push(previewWebsitePage.current)
job_id: websiteCrawlJobId,
result: previewWebsitePage.current,
}
datasourceInfoList.push(documentInfo)
}
await runPublishedPipeline({ await runPublishedPipeline({
pipeline_id: pipelineId!, pipeline_id: pipelineId!,
inputs: data, inputs: data,
@ -143,7 +138,7 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
}, },
}) })
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId]) }, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
const handleProcess = useCallback(async (data: Record<string, any>) => { const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource) if (!datasource)
@ -176,11 +171,9 @@ const CreateFormPipeline = () => {
}) })
} }
if (datasource.type === DatasourceType.websiteCrawl) { if (datasource.type === DatasourceType.websiteCrawl) {
const documentInfo = { websitePages.forEach((websitePage) => {
job_id: websiteCrawlJobId, datasourceInfoList.push(websitePage)
result: websitePages, })
}
datasourceInfoList.push(documentInfo)
} }
await runPublishedPipeline({ await runPublishedPipeline({
pipeline_id: pipelineId!, pipeline_id: pipelineId!,
@ -196,7 +189,7 @@ const CreateFormPipeline = () => {
handleNextStep() handleNextStep()
}, },
}) })
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages]) }, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
const onClickProcess = useCallback(() => { const onClickProcess = useCallback(() => {
isPreview.current = false isPreview.current = false
@ -285,7 +278,7 @@ const CreateFormPipeline = () => {
docLink: datasource.docLink || '', docLink: datasource.docLink || '',
}} }}
checkedCrawlResult={websitePages} checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages} onCheckedCrawlResultChange={updataCheckedCrawlResultChange}
onJobIdChange={setWebsiteCrawlJobId} onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite} onPreview={updateCurrentWebsite}
/> />

4
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result.tsx
Unescape Escape View File

@ -62,14 +62,14 @@ const CrawledResult = ({
time: usedTime.toFixed(1), time: usedTime.toFixed(1),
})} })}
</div> </div>
<div className='rounded-xl border border-components-panel-border bg-components-panel-bg'> <div className='overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg'>
<div className='flex items-center px-4 py-2'> <div className='flex items-center px-4 py-2'>
<CheckboxWithLabel <CheckboxWithLabel
isChecked={isCheckAll} isChecked={isCheckAll}
onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)} onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
/> />
</div> </div>
<div className='flex flex-col gap-y-px overflow-hidden border-t border-divider-subtle bg-background-default-subtle p-2'> <div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
{list.map((item, index) => ( {list.map((item, index) => (
<CrawledResultItem <CrawledResultItem
key={item.source_url} key={item.source_url}

4
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
Unescape Escape View File

@ -11,7 +11,7 @@ import {
useDraftDatasourceNodeRun, useDraftDatasourceNodeRun,
useDraftPipelinePreProcessingParams, useDraftPipelinePreProcessingParams,
usePublishedDatasourceNodeRun, usePublishedDatasourceNodeRun,
usePublishedPipelineProcessingParams, usePublishedPipelinePreProcessingParams,
} from '@/service/use-pipeline' } from '@/service/use-pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { DatasourceType } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline'
@ -52,7 +52,7 @@ const Crawler = ({
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0) const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelineProcessingParams : useDraftPipelinePreProcessingParams) const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
const { data: paramsConfig } = usePreProcessingParams.current({ const { data: paramsConfig } = usePreProcessingParams.current({
pipeline_id: pipelineId!, pipeline_id: pipelineId!,
node_id: nodeId, node_id: nodeId,

13
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
Unescape Escape View File

@ -40,7 +40,7 @@ const TestRunPanel = () => {
} = useOnlineDocuments() } = useOnlineDocuments()
const { const {
websitePages, websitePages,
websiteCrawlJobId, // websiteCrawlJobId, // todo: Add status query
setWebsitePages, setWebsitePages,
setWebsiteCrawlJobId, setWebsiteCrawlJobId,
} = useWebsiteCrawl() } = useWebsiteCrawl()
@ -90,20 +90,15 @@ const TestRunPanel = () => {
} }
datasourceInfoList.push(documentInfo) datasourceInfoList.push(documentInfo)
} }
if (datasource.type === DatasourceType.websiteCrawl) { if (datasource.type === DatasourceType.websiteCrawl)
const documentInfo = { datasourceInfoList.push(websitePages[0])
job_id: websiteCrawlJobId,
result: [websitePages[0]],
}
datasourceInfoList.push(documentInfo)
}
handleRun({ handleRun({
inputs: data, inputs: data,
start_node_id: datasource.nodeId, start_node_id: datasource.nodeId,
datasource_type: datasource.type, datasource_type: datasource.type,
datasource_info_list: datasourceInfoList, datasource_info_list: datasourceInfoList,
}) })
}, [datasource, fileList, handleRun, onlineDocuments, websiteCrawlJobId, websitePages]) }, [datasource, fileList, handleRun, onlineDocuments, websitePages])
return ( return (
<div <div

2
web/service/use-pipeline.ts
Unescape Escape View File

@ -288,7 +288,7 @@ export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProce
return useQuery<PipelinePreProcessingParamsResponse>({ return useQuery<PipelinePreProcessingParamsResponse>({
queryKey: [NAME_SPACE, 'published-pipeline-pre-processing-params', pipeline_id, node_id], queryKey: [NAME_SPACE, 'published-pipeline-pre-processing-params', pipeline_id, node_id],
queryFn: () => { queryFn: () => {
return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, { return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/pre-processing/parameters`, {
params: { params: {
node_id, node_id,
}, },

Write Preview
Loading…
Cancel
Save