feat: Implement data source store with slices for local files, online documents, website crawls, and online drives

web/app/components/datasets/documents/create-from-pipeline/data-source/store/index.ts

@@ -0,0 +1,34 @@
+import { useContext } from 'react'
+import { createStore, useStore } from 'zustand'
+import { DataSourceContext } from './provider'
+import type { LocalFileSliceShape } from './slices/local-file'
+import { createLocalFileSlice } from './slices/local-file'
+import type { OnlineDocumentSliceShape } from './slices/online-document'
+import { createOnlineDocumentSlice } from './slices/online-document'
+import type { WebsiteCrawlSliceShape } from './slices/website-crawl'
+import { createWebsiteCrawlSlice } from './slices/website-crawl'
+import type { OnlineDriveSliceShape } from './slices/online-drive'
+import { createOnlineDriveSlice } from './slices/online-drive'
+
+export type DataSourceShape =
+  LocalFileSliceShape &
+  OnlineDocumentSliceShape &
+  WebsiteCrawlSliceShape &
+  OnlineDriveSliceShape
+
+export const createDataSourceStore = () => {
+  return createStore<DataSourceShape>((...args) => ({
+    ...createLocalFileSlice(...args),
+    ...createOnlineDocumentSlice(...args),
+    ...createWebsiteCrawlSlice(...args),
+    ...createOnlineDriveSlice(...args),
+  }))
+}
+
+export const useDataSourceStore = <T>(selector: (state: DataSourceShape) => T): T => {
+  const store = useContext(DataSourceContext)
+  if (!store)
+    throw new Error('Missing DataSourceContext.Provider in the tree')
+
+  return useStore(store, selector)
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/provider.tsx

@@ -0,0 +1,29 @@
+import { createContext, useRef } from 'react'
+import { createDataSourceStore } from './'
+
+type DataSourceStoreApi = ReturnType<typeof createDataSourceStore>
+
+type DataSourceContextType = DataSourceStoreApi | null
+
+export const DataSourceContext = createContext<DataSourceContextType>(null)
+
+type DataSourceProviderProps = {
+  children: React.ReactNode
+}
+
+const DataSourceProvider = ({
+  children,
+}: DataSourceProviderProps) => {
+  const storeRef = useRef<DataSourceStoreApi>()
+
+  if (!storeRef.current)
+    storeRef.current = createDataSourceStore()
+
+  return (
+    <DataSourceContext.Provider value={storeRef.current!}>
+      {children}
+    </DataSourceContext.Provider>
+  )
+}
+
+export default DataSourceProvider

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts

@@ -0,0 +1,22 @@
+import type { StateCreator } from 'zustand'
+import type { FileItem } from '@/models/datasets'
+
+export type LocalFileSliceShape = {
+  localFileList: FileItem[]
+  setLocalFileList: (fileList: FileItem[]) => void
+  currentLocalFile: File | undefined
+  setCurrentLocalFile: (file: File | undefined) => void
+}
+
+export const createLocalFileSlice: StateCreator<LocalFileSliceShape> = (set) => {
+  return ({
+    localFileList: [],
+    setLocalFileList: (fileList: FileItem[]) => set(() => ({
+      localFileList: fileList,
+    })),
+    currentLocalFile: undefined,
+    setCurrentLocalFile: (file: File | undefined) => set(() => ({
+      currentLocalFile: file,
+    })),
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts

@@ -0,0 +1,46 @@
+import type { StateCreator } from 'zustand'
+import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common'
+
+export type OnlineDocumentSliceShape = {
+  documentData: DataSourceNotionWorkspace[]
+  setDocumentData: (documentData: DataSourceNotionWorkspace[]) => void
+  searchValue: string
+  setSearchValue: (searchValue: string) => void
+  currentWorkspaceId: string
+  setCurrentWorkspaceId: (workspaceId: string) => void
+  onlineDocuments: NotionPage[]
+  setOnlineDocuments: (documents: NotionPage[]) => void
+  currentDocument: NotionPage | undefined
+  setCurrentDocument: (document: NotionPage | undefined) => void
+  selectedPagesId: Set<string>
+  setSelectedPagesId: (selectedPagesId: Set<string>) => void
+}
+
+export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set, get) => {
+  return ({
+    documentData: [],
+    setDocumentData: (documentData: DataSourceNotionWorkspace[]) => set(() => ({
+      documentData,
+    })),
+    searchValue: '',
+    setSearchValue: (searchValue: string) => set(() => ({
+      searchValue,
+    })),
+    currentWorkspaceId: '',
+    setCurrentWorkspaceId: (workspaceId: string) => set(() => ({
+      currentWorkspaceId: workspaceId,
+    })),
+    onlineDocuments: [],
+    setOnlineDocuments: (documents: NotionPage[]) => set(() => ({
+      onlineDocuments: documents,
+    })),
+    currentDocument: undefined,
+    setCurrentDocument: (document: NotionPage | undefined) => set(() => ({
+      currentDocument: document,
+    })),
+    selectedPagesId: new Set([...get().onlineDocuments.map(doc => doc.page_id)]),
+    setSelectedPagesId: (selectedPagesId: Set<string>) => set(() => ({
+      selectedPagesId,
+    })),
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts

@@ -0,0 +1,44 @@
+import type { StateCreator } from 'zustand'
+import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline'
+
+export type OnlineDriveSliceShape = {
+  prefix: string[]
+  setPrefix: (prefix: string[]) => void
+  keywords: string
+  setKeywords: (keywords: string) => void
+  startAfter: string
+  setStartAfter: (startAfter: string) => void
+  selectedFileList: string[]
+  setSelectedFileList: (selectedFileList: string[]) => void
+  fileList: OnlineDriveFile[]
+  setFileList: (fileList: OnlineDriveFile[]) => void
+}
+
+export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set) => {
+  return ({
+    prefix: [],
+    setPrefix: (prefix: string[]) => set(() => ({
+      prefix,
+    })),
+    keywords: '',
+    setKeywords: (keywords: string) => set(() => ({
+      keywords,
+    })),
+    startAfter: '',
+    setStartAfter: (startAfter: string) => set(() => ({
+      startAfter,
+    })),
+    selectedFileList: [],
+    setSelectedFileList: (selectedFileList: string[]) => set(() => ({
+      selectedFileList,
+    })),
+    fileList: [{
+      key: 'Bucket_1',
+      size: 1024, // unit bytes
+      type: OnlineDriveFileType.bucket,
+    }],
+    setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
+      fileList,
+    })),
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts

@@ -0,0 +1,41 @@
+import type { StateCreator } from 'zustand'
+import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
+import { CrawlStep } from '@/models/datasets'
+
+export type WebsiteCrawlSliceShape = {
+  websitePages: CrawlResultItem[]
+  setWebsitePages: (pages: CrawlResultItem[]) => void
+  currentWebsite: CrawlResultItem | undefined
+  setCurrentWebsite: (website: CrawlResultItem | undefined) => void
+  crawlResult: CrawlResult | undefined
+  setCrawlResult: (result: CrawlResult | undefined) => void
+  step: CrawlStep
+  setStep: (step: CrawlStep) => void
+  previewIndex: number
+  setPreviewIndex: (index: number) => void
+}
+
+export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set) => {
+  return ({
+    websitePages: [],
+    setWebsitePages: (pages: CrawlResultItem[]) => set(() => ({
+      websitePages: pages,
+    })),
+    currentWebsite: undefined,
+    setCurrentWebsite: (website: CrawlResultItem | undefined) => set(() => ({
+      currentWebsite: website,
+    })),
+    crawlResult: undefined,
+    setCrawlResult: (result: CrawlResult | undefined) => set(() => ({
+      crawlResult: result,
+    })),
+    step: CrawlStep.init,
+    setStep: (step: CrawlStep) => set(() => ({
+      step,
+    })),
+    previewIndex: -1,
+    setPreviewIndex: (index: number) => set(() => ({
+      previewIndex: index,
+    })),
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -28,6 +28,7 @@ import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, Publis
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
 import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
+import DataSourceProvider from './data-source/store/provider'
 
 const CreateFormPipeline = () => {
   const { t } = useTranslation()
@@ -399,4 +400,12 @@ const CreateFormPipeline = () => {
   )
 }
 
-export default CreateFormPipeline
+const CreateFormPipelineWrapper = () => {
+  return (
+    <DataSourceProvider>
+      <CreateFormPipeline />
+    </DataSourceProvider>
+  )
+}
+
+export default CreateFormPipelineWrapper

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -15,6 +15,7 @@ import { TransferMethod } from '@/types/app'
 import CloseButton from './close-button'
 import Header from './header'
 import FooterTips from './footer-tips'
+import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider'
 
 const TestRunPanel = () => {
   const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
@@ -209,4 +210,12 @@ const TestRunPanel = () => {
   )
 }
 
-export default TestRunPanel
+const TestRunPanelWrapper = () => {
+  return (
+    <DataSourceProvider>
+      <TestRunPanel />
+    </DataSourceProvider>
+  )
+}
+
+export default TestRunPanelWrapper

web/i18n/en-US/dataset-pipeline.ts

@@ -10,8 +10,9 @@ const translation = {
       description: 'Import from a DSL file',
     },
     createKnowledge: 'Create Knowledge',
-    errorTip: 'Failed to create a Knowledge Pipeline',
+    errorTip: 'Failed to create a Knowledge Base',
-    successTip: 'Successfully created a Knowledge Pipeline',
+    successTip: 'Successfully created a Knowledge Base',
+    caution: 'Caution',
   },
   tabs: {
     builtInPipeline: 'Built-in pipeline',

web/i18n/zh-Hans/dataset-pipeline.ts

@@ -1,20 +1,21 @@
 const translation = {
   creation: {
-    title: '创建知识库流水线',
+    title: '创建知识库 pipeline',
     createFromScratch: {
       title: '从零开始创建',
-      description: '空白知识库流水线',
+      description: '空白知识库 pipeline',
     },
     ImportDSL: {
       title: '导入',
       description: '从 DSL 文件导入',
     },
     createKnowledge: '创建知识库',
-    errorTip: '创建知识库流水线失败',
+    errorTip: '创建知识库',
-    successTip: '成功创建知识库流水线',
+    successTip: '成功创建知识库',
+    caution: '注意',
   },
   tabs: {
-    builtInPipeline: '内置流水线',
+    builtInPipeline: '内置 pipeline',
     customized: '自定义',
   },
   operations: {
@@ -22,7 +23,7 @@ const translation = {
     details: '详情',
     editInfo: '编辑信息',
     exportDSL: '导出 DSL',
-    useTemplate: '使用此知识库流水线',
+    useTemplate: '使用此知识库 pipeline',
     backToDataSource: '返回数据源',
     process: '处理',
     dataSource: '数据源',
@@ -34,15 +35,15 @@ const translation = {
   knowledgeDescription: '知识库描述',
   knowledgeDescriptionPlaceholder: '描述知识库中的内容。详细的描述可以让 AI 更准确地访问数据集的内容。如果为空，Dify 将使用默认的命中策略。（可选）',
   knowledgePermissions: '权限',
-  editPipelineInfo: '编辑流水线信息',
+  editPipelineInfo: '编辑 pipeline 信息',
-  pipelineNameAndIcon: '流水线名称和图标',
+  pipelineNameAndIcon: 'pipeline 名称和图标',
   deletePipeline: {
-    title: '要删除此流水线模板吗？',
+    title: '要删除此 pipeline 模板吗？',
-    content: '删除流水线模板是不可逆的。',
+    content: '删除 pipeline 模板是不可逆的。',
   },
   exportDSL: {
-    successTip: '成功导出流水线 DSL',
+    successTip: '成功导出 pipeline DSL',
-    errorTip: '导出流水线 DSL 失败',
+    errorTip: '导出 pipeline DSL 失败',
   },
   details: {
     createdBy: '由 {{author}} 创建',
@@ -67,7 +68,7 @@ const translation = {
   inputField: '输入字段',
   inputFieldPanel: {
     title: '用户输入字段',
-    description: '用户输入字段用于定义和收集流水线执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。',
+    description: '用户输入字段用于定义和收集 pipeline 执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。',
     uniqueInputs: {
       title: '非共享输入',
       tooltip: '非共享输入只能被选定的数据源及其下游节点访问。用户在选择其他数据源时不需要填写它。只有数据源变量引用的输入字段才会出现在第一步（数据源）中。所有其他字段将在第二步（Process Documents）中显示。',