From 2dd1f41ad3e9e8114d74928f55518fb03f60449a Mon Sep 17 00:00:00 2001 From: twwu Date: Thu, 3 Jul 2025 10:31:29 +0800 Subject: [PATCH] feat: Implement data source store with slices for local files, online documents, website crawls, and online drives --- .../data-source/store/index.ts | 34 ++++++++++++++ .../data-source/store/provider.tsx | 29 ++++++++++++ .../data-source/store/slices/local-file.ts | 22 +++++++++ .../store/slices/online-document.ts | 46 +++++++++++++++++++ .../data-source/store/slices/online-drive.ts | 44 ++++++++++++++++++ .../data-source/store/slices/website-crawl.ts | 41 +++++++++++++++++ .../documents/create-from-pipeline/index.tsx | 11 ++++- .../components/panel/test-run/index.tsx | 11 ++++- web/i18n/en-US/dataset-pipeline.ts | 5 +- web/i18n/zh-Hans/dataset-pipeline.ts | 27 +++++------ 10 files changed, 253 insertions(+), 17 deletions(-) create mode 100644 web/app/components/datasets/documents/create-from-pipeline/data-source/store/index.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/data-source/store/provider.tsx create mode 100644 web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts create mode 100644 web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/index.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/index.ts new file mode 100644 index 0000000000..1843e38c48 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/index.ts @@ -0,0 +1,34 @@ +import { useContext } from 'react' +import { createStore, useStore } from 'zustand' +import { DataSourceContext } from './provider' +import type { LocalFileSliceShape } from './slices/local-file' +import { createLocalFileSlice } from './slices/local-file' +import type { OnlineDocumentSliceShape } from './slices/online-document' +import { createOnlineDocumentSlice } from './slices/online-document' +import type { WebsiteCrawlSliceShape } from './slices/website-crawl' +import { createWebsiteCrawlSlice } from './slices/website-crawl' +import type { OnlineDriveSliceShape } from './slices/online-drive' +import { createOnlineDriveSlice } from './slices/online-drive' + +export type DataSourceShape = + LocalFileSliceShape & + OnlineDocumentSliceShape & + WebsiteCrawlSliceShape & + OnlineDriveSliceShape + +export const createDataSourceStore = () => { + return createStore((...args) => ({ + ...createLocalFileSlice(...args), + ...createOnlineDocumentSlice(...args), + ...createWebsiteCrawlSlice(...args), + ...createOnlineDriveSlice(...args), + })) +} + +export const useDataSourceStore = (selector: (state: DataSourceShape) => T): T => { + const store = useContext(DataSourceContext) + if (!store) + throw new Error('Missing DataSourceContext.Provider in the tree') + + return useStore(store, selector) +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/provider.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/provider.tsx new file mode 100644 index 0000000000..446d47704c --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/provider.tsx @@ -0,0 +1,29 @@ +import { createContext, useRef } from 'react' +import { createDataSourceStore } from './' + +type DataSourceStoreApi = ReturnType + +type DataSourceContextType = DataSourceStoreApi | null + +export const DataSourceContext = createContext(null) + +type DataSourceProviderProps = { + children: React.ReactNode +} + +const DataSourceProvider = ({ + children, +}: DataSourceProviderProps) => { + const storeRef = useRef() + + if (!storeRef.current) + storeRef.current = createDataSourceStore() + + return ( + + {children} + + ) +} + +export default DataSourceProvider diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts new file mode 100644 index 0000000000..b073827c57 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts @@ -0,0 +1,22 @@ +import type { StateCreator } from 'zustand' +import type { FileItem } from '@/models/datasets' + +export type LocalFileSliceShape = { + localFileList: FileItem[] + setLocalFileList: (fileList: FileItem[]) => void + currentLocalFile: File | undefined + setCurrentLocalFile: (file: File | undefined) => void +} + +export const createLocalFileSlice: StateCreator = (set) => { + return ({ + localFileList: [], + setLocalFileList: (fileList: FileItem[]) => set(() => ({ + localFileList: fileList, + })), + currentLocalFile: undefined, + setCurrentLocalFile: (file: File | undefined) => set(() => ({ + currentLocalFile: file, + })), + }) +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts new file mode 100644 index 0000000000..77a9eb5851 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts @@ -0,0 +1,46 @@ +import type { StateCreator } from 'zustand' +import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common' + +export type OnlineDocumentSliceShape = { + documentData: DataSourceNotionWorkspace[] + setDocumentData: (documentData: DataSourceNotionWorkspace[]) => void + searchValue: string + setSearchValue: (searchValue: string) => void + currentWorkspaceId: string + setCurrentWorkspaceId: (workspaceId: string) => void + onlineDocuments: NotionPage[] + setOnlineDocuments: (documents: NotionPage[]) => void + currentDocument: NotionPage | undefined + setCurrentDocument: (document: NotionPage | undefined) => void + selectedPagesId: Set + setSelectedPagesId: (selectedPagesId: Set) => void +} + +export const createOnlineDocumentSlice: StateCreator = (set, get) => { + return ({ + documentData: [], + setDocumentData: (documentData: DataSourceNotionWorkspace[]) => set(() => ({ + documentData, + })), + searchValue: '', + setSearchValue: (searchValue: string) => set(() => ({ + searchValue, + })), + currentWorkspaceId: '', + setCurrentWorkspaceId: (workspaceId: string) => set(() => ({ + currentWorkspaceId: workspaceId, + })), + onlineDocuments: [], + setOnlineDocuments: (documents: NotionPage[]) => set(() => ({ + onlineDocuments: documents, + })), + currentDocument: undefined, + setCurrentDocument: (document: NotionPage | undefined) => set(() => ({ + currentDocument: document, + })), + selectedPagesId: new Set([...get().onlineDocuments.map(doc => doc.page_id)]), + setSelectedPagesId: (selectedPagesId: Set) => set(() => ({ + selectedPagesId, + })), + }) +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts new file mode 100644 index 0000000000..83808e26e2 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts @@ -0,0 +1,44 @@ +import type { StateCreator } from 'zustand' +import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline' + +export type OnlineDriveSliceShape = { + prefix: string[] + setPrefix: (prefix: string[]) => void + keywords: string + setKeywords: (keywords: string) => void + startAfter: string + setStartAfter: (startAfter: string) => void + selectedFileList: string[] + setSelectedFileList: (selectedFileList: string[]) => void + fileList: OnlineDriveFile[] + setFileList: (fileList: OnlineDriveFile[]) => void +} + +export const createOnlineDriveSlice: StateCreator = (set) => { + return ({ + prefix: [], + setPrefix: (prefix: string[]) => set(() => ({ + prefix, + })), + keywords: '', + setKeywords: (keywords: string) => set(() => ({ + keywords, + })), + startAfter: '', + setStartAfter: (startAfter: string) => set(() => ({ + startAfter, + })), + selectedFileList: [], + setSelectedFileList: (selectedFileList: string[]) => set(() => ({ + selectedFileList, + })), + fileList: [{ + key: 'Bucket_1', + size: 1024, // unit bytes + type: OnlineDriveFileType.bucket, + }], + setFileList: (fileList: OnlineDriveFile[]) => set(() => ({ + fileList, + })), + }) +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts new file mode 100644 index 0000000000..715f85d461 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts @@ -0,0 +1,41 @@ +import type { StateCreator } from 'zustand' +import type { CrawlResult, CrawlResultItem } from '@/models/datasets' +import { CrawlStep } from '@/models/datasets' + +export type WebsiteCrawlSliceShape = { + websitePages: CrawlResultItem[] + setWebsitePages: (pages: CrawlResultItem[]) => void + currentWebsite: CrawlResultItem | undefined + setCurrentWebsite: (website: CrawlResultItem | undefined) => void + crawlResult: CrawlResult | undefined + setCrawlResult: (result: CrawlResult | undefined) => void + step: CrawlStep + setStep: (step: CrawlStep) => void + previewIndex: number + setPreviewIndex: (index: number) => void +} + +export const createWebsiteCrawlSlice: StateCreator = (set) => { + return ({ + websitePages: [], + setWebsitePages: (pages: CrawlResultItem[]) => set(() => ({ + websitePages: pages, + })), + currentWebsite: undefined, + setCurrentWebsite: (website: CrawlResultItem | undefined) => set(() => ({ + currentWebsite: website, + })), + crawlResult: undefined, + setCrawlResult: (result: CrawlResult | undefined) => set(() => ({ + crawlResult: result, + })), + step: CrawlStep.init, + setStep: (step: CrawlStep) => set(() => ({ + step, + })), + previewIndex: -1, + setPreviewIndex: (index: number) => set(() => ({ + previewIndex: index, + })), + }) +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index f86b0b13ea..3a7f237a1a 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -28,6 +28,7 @@ import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, Publis import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks' +import DataSourceProvider from './data-source/store/provider' const CreateFormPipeline = () => { const { t } = useTranslation() @@ -399,4 +400,12 @@ const CreateFormPipeline = () => { ) } -export default CreateFormPipeline +const CreateFormPipelineWrapper = () => { + return ( + + + + ) +} + +export default CreateFormPipelineWrapper diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index 71699a7a6f..148ba7e636 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -15,6 +15,7 @@ import { TransferMethod } from '@/types/app' import CloseButton from './close-button' import Header from './header' import FooterTips from './footer-tips' +import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider' const TestRunPanel = () => { const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel) @@ -209,4 +210,12 @@ const TestRunPanel = () => { ) } -export default TestRunPanel +const TestRunPanelWrapper = () => { + return ( + + + + ) +} + +export default TestRunPanelWrapper diff --git a/web/i18n/en-US/dataset-pipeline.ts b/web/i18n/en-US/dataset-pipeline.ts index 5a55bdeec1..d35d67c618 100644 --- a/web/i18n/en-US/dataset-pipeline.ts +++ b/web/i18n/en-US/dataset-pipeline.ts @@ -10,8 +10,9 @@ const translation = { description: 'Import from a DSL file', }, createKnowledge: 'Create Knowledge', - errorTip: 'Failed to create a Knowledge Pipeline', - successTip: 'Successfully created a Knowledge Pipeline', + errorTip: 'Failed to create a Knowledge Base', + successTip: 'Successfully created a Knowledge Base', + caution: 'Caution', }, tabs: { builtInPipeline: 'Built-in pipeline', diff --git a/web/i18n/zh-Hans/dataset-pipeline.ts b/web/i18n/zh-Hans/dataset-pipeline.ts index 053780d222..0c2c18dd42 100644 --- a/web/i18n/zh-Hans/dataset-pipeline.ts +++ b/web/i18n/zh-Hans/dataset-pipeline.ts @@ -1,20 +1,21 @@ const translation = { creation: { - title: '创建知识库流水线', + title: '创建知识库 pipeline', createFromScratch: { title: '从零开始创建', - description: '空白知识库流水线', + description: '空白知识库 pipeline', }, ImportDSL: { title: '导入', description: '从 DSL 文件导入', }, createKnowledge: '创建知识库', - errorTip: '创建知识库流水线失败', - successTip: '成功创建知识库流水线', + errorTip: '创建知识库', + successTip: '成功创建知识库', + caution: '注意', }, tabs: { - builtInPipeline: '内置流水线', + builtInPipeline: '内置 pipeline', customized: '自定义', }, operations: { @@ -22,7 +23,7 @@ const translation = { details: '详情', editInfo: '编辑信息', exportDSL: '导出 DSL', - useTemplate: '使用此知识库流水线', + useTemplate: '使用此知识库 pipeline', backToDataSource: '返回数据源', process: '处理', dataSource: '数据源', @@ -34,15 +35,15 @@ const translation = { knowledgeDescription: '知识库描述', knowledgeDescriptionPlaceholder: '描述知识库中的内容。详细的描述可以让 AI 更准确地访问数据集的内容。如果为空，Dify 将使用默认的命中策略。（可选）', knowledgePermissions: '权限', - editPipelineInfo: '编辑流水线信息', - pipelineNameAndIcon: '流水线名称和图标', + editPipelineInfo: '编辑 pipeline 信息', + pipelineNameAndIcon: 'pipeline 名称和图标', deletePipeline: { - title: '要删除此流水线模板吗？', - content: '删除流水线模板是不可逆的。', + title: '要删除此 pipeline 模板吗？', + content: '删除 pipeline 模板是不可逆的。', }, exportDSL: { - successTip: '成功导出流水线 DSL', - errorTip: '导出流水线 DSL 失败', + successTip: '成功导出 pipeline DSL', + errorTip: '导出 pipeline DSL 失败', }, details: { createdBy: '由 {{author}} 创建', @@ -67,7 +68,7 @@ const translation = { inputField: '输入字段', inputFieldPanel: { title: '用户输入字段', - description: '用户输入字段用于定义和收集流水线执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。', + description: '用户输入字段用于定义和收集 pipeline 执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。', uniqueInputs: { title: '非共享输入', tooltip: '非共享输入只能被选定的数据源及其下游节点访问。用户在选择其他数据源时不需要填写它。只有数据源变量引用的输入字段才会出现在第一步（数据源）中。所有其他字段将在第二步（Process Documents）中显示。',