feat: implement document settings and pipeline settings components with localization support
twwu
10 months ago
parent
a7eb534761
commit
141d6b1abf
@ -0,0 +1,94 @@
|
||||
import React, { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { useRouter } from 'next/navigation'
|
||||
import DatasetDetailContext from '@/context/dataset-detail'
|
||||
import type { CrawlOptions, CustomFile, DataSourceType } from '@/models/datasets'
|
||||
import Loading from '@/app/components/base/loading'
|
||||
import StepTwo from '@/app/components/datasets/create/step-two'
|
||||
import AccountSetting from '@/app/components/header/account-setting'
|
||||
import AppUnavailable from '@/app/components/base/app-unavailable'
|
||||
import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
|
||||
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import { useDocumentDetail, useInvalidDocumentDetailKey } from '@/service/knowledge/use-document'
|
||||
|
||||
type DocumentSettingsProps = {
|
||||
datasetId: string
|
||||
documentId: string
|
||||
}
|
||||
|
||||
const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
|
||||
const { t } = useTranslation()
|
||||
const router = useRouter()
|
||||
const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
|
||||
const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
|
||||
const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
|
||||
|
||||
const invalidDocumentDetail = useInvalidDocumentDetailKey()
|
||||
const saveHandler = () => {
|
||||
invalidDocumentDetail()
|
||||
router.push(`/datasets/${datasetId}/documents/${documentId}`)
|
||||
}
|
||||
|
||||
const cancelHandler = () => router.back()
|
||||
|
||||
const { data: documentDetail, error } = useDocumentDetail({
|
||||
datasetId,
|
||||
documentId,
|
||||
params: { metadata: 'without' },
|
||||
})
|
||||
|
||||
const currentPage = useMemo(() => {
|
||||
return {
|
||||
workspace_id: documentDetail?.data_source_info.notion_workspace_id,
|
||||
page_id: documentDetail?.data_source_info.notion_page_id,
|
||||
page_name: documentDetail?.name,
|
||||
page_icon: documentDetail?.data_source_info.notion_page_icon,
|
||||
type: documentDetail?.data_source_type,
|
||||
}
|
||||
}, [documentDetail])
|
||||
|
||||
if (error)
|
||||
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
|
||||
|
||||
return (
|
||||
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
|
||||
<div className='grow'>
|
||||
{!documentDetail && <Loading type='app' />}
|
||||
{dataset && documentDetail && (
|
||||
<StepTwo
|
||||
isAPIKeySet={!!embeddingsDefaultModel}
|
||||
onSetting={showSetAPIKey}
|
||||
datasetId={datasetId}
|
||||
dataSourceType={documentDetail.data_source_type as DataSourceType}
|
||||
notionPages={[currentPage as unknown as NotionPage]}
|
||||
websitePages={[
|
||||
{
|
||||
title: documentDetail.name,
|
||||
source_url: documentDetail.data_source_info?.url,
|
||||
markdown: '',
|
||||
description: '',
|
||||
},
|
||||
]}
|
||||
websiteCrawlProvider={documentDetail.data_source_info?.provider}
|
||||
websiteCrawlJobId={documentDetail.data_source_info?.job_id}
|
||||
crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
|
||||
indexingType={indexingTechnique}
|
||||
isSetting
|
||||
documentDetail={documentDetail}
|
||||
files={[documentDetail.data_source_info.upload_file as CustomFile]}
|
||||
onSave={saveHandler}
|
||||
onCancel={cancelHandler}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
{isShowSetAPIKey && <AccountSetting activeTab='provider' onCancel={async () => {
|
||||
hideSetAPIkey()
|
||||
}} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default DocumentSettings
|
||||
@ -1,96 +1,35 @@
|
||||
'use client'
|
||||
import React, { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { useRouter } from 'next/navigation'
|
||||
import DatasetDetailContext from '@/context/dataset-detail'
|
||||
import type { CrawlOptions, CustomFile } from '@/models/datasets'
|
||||
import React from 'react'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import DocumentSettings from './document-settings'
|
||||
import PipelineSettings from './pipeline-settings'
|
||||
|
||||
import Loading from '@/app/components/base/loading'
|
||||
import StepTwo from '@/app/components/datasets/create/step-two'
|
||||
import AccountSetting from '@/app/components/header/account-setting'
|
||||
import AppUnavailable from '@/app/components/base/app-unavailable'
|
||||
import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
|
||||
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import { useDocumentDetail, useInvalidDocumentDetailKey } from '@/service/knowledge/use-document'
|
||||
|
||||
type DocumentSettingsProps = {
|
||||
type SettingsProps = {
|
||||
datasetId: string
|
||||
documentId: string
|
||||
}
|
||||
|
||||
const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
|
||||
const { t } = useTranslation()
|
||||
const router = useRouter()
|
||||
const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
|
||||
const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
|
||||
const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
|
||||
|
||||
const invalidDocumentDetail = useInvalidDocumentDetailKey()
|
||||
const saveHandler = () => {
|
||||
invalidDocumentDetail()
|
||||
router.push(`/datasets/${datasetId}/documents/${documentId}`)
|
||||
const Settings = ({
|
||||
datasetId,
|
||||
documentId,
|
||||
}: SettingsProps) => {
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
|
||||
if (!pipelineId) {
|
||||
return (
|
||||
<DocumentSettings
|
||||
datasetId={datasetId}
|
||||
documentId={documentId}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
const cancelHandler = () => router.back()
|
||||
|
||||
const { data: documentDetail, error } = useDocumentDetail({
|
||||
datasetId,
|
||||
documentId,
|
||||
params: { metadata: 'without' },
|
||||
})
|
||||
|
||||
const currentPage = useMemo(() => {
|
||||
return {
|
||||
workspace_id: documentDetail?.data_source_info.notion_workspace_id,
|
||||
page_id: documentDetail?.data_source_info.notion_page_id,
|
||||
page_name: documentDetail?.name,
|
||||
page_icon: documentDetail?.data_source_info.notion_page_icon,
|
||||
type: documentDetail?.data_source_type,
|
||||
}
|
||||
}, [documentDetail])
|
||||
|
||||
if (error)
|
||||
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
|
||||
|
||||
return (
|
||||
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
|
||||
<div className="grow">
|
||||
{!documentDetail && <Loading type='app' />}
|
||||
{dataset && documentDetail && (
|
||||
<StepTwo
|
||||
isAPIKeySet={!!embeddingsDefaultModel}
|
||||
onSetting={showSetAPIKey}
|
||||
datasetId={datasetId}
|
||||
dataSourceType={documentDetail.data_source_type}
|
||||
notionPages={[currentPage as unknown as NotionPage]}
|
||||
websitePages={[
|
||||
{
|
||||
title: documentDetail.name,
|
||||
source_url: documentDetail.data_source_info?.url,
|
||||
markdown: '',
|
||||
description: '',
|
||||
},
|
||||
]}
|
||||
websiteCrawlProvider={documentDetail.data_source_info?.provider}
|
||||
websiteCrawlJobId={documentDetail.data_source_info?.job_id}
|
||||
crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
|
||||
indexingType={indexingTechnique}
|
||||
isSetting
|
||||
documentDetail={documentDetail}
|
||||
files={[documentDetail.data_source_info.upload_file as CustomFile]}
|
||||
onSave={saveHandler}
|
||||
onCancel={cancelHandler}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
{isShowSetAPIKey && <AccountSetting activeTab="provider" onCancel={async () => {
|
||||
hideSetAPIkey()
|
||||
}} />}
|
||||
</div>
|
||||
<PipelineSettings
|
||||
datasetId={datasetId}
|
||||
documentId={documentId}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export default DocumentSettings
|
||||
export default Settings
|
||||
|
||||
@ -0,0 +1,120 @@
|
||||
import { useCallback, useRef, useState } from 'react'
|
||||
import type { CrawlResultItem, DocumentItem, FileIndexingEstimateResponse } from '@/models/datasets'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { useDocumentDetail } from '@/service/knowledge/use-document'
|
||||
import AppUnavailable from '@/app/components/base/app-unavailable'
|
||||
import ChunkPreview from '../../../create-from-pipeline/preview/chunk-preview'
|
||||
import Loading from '@/app/components/base/loading'
|
||||
import type { DatasourceType } from '@/models/pipeline'
|
||||
import ProcessDocuments from './process-documents'
|
||||
import LeftHeader from './left-header'
|
||||
|
||||
type PipelineSettingsProps = {
|
||||
datasetId: string
|
||||
documentId: string
|
||||
}
|
||||
|
||||
const PipelineSettings = ({
|
||||
datasetId,
|
||||
documentId,
|
||||
}: PipelineSettingsProps) => {
|
||||
const { t } = useTranslation()
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
|
||||
|
||||
const isPreview = useRef(false)
|
||||
const formRef = useRef<any>(null)
|
||||
|
||||
const { data: documentDetail, error, isFetching: isFetchingDocumentDetail } = useDocumentDetail({
|
||||
datasetId,
|
||||
documentId,
|
||||
params: { metadata: 'without' },
|
||||
})
|
||||
|
||||
const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
|
||||
// todo: Preview
|
||||
}, [])
|
||||
|
||||
const handleProcess = useCallback(async (data: Record<string, any>) => {
|
||||
// todo: Process
|
||||
}, [])
|
||||
|
||||
const onClickProcess = useCallback(() => {
|
||||
isPreview.current = false
|
||||
formRef.current?.submit()
|
||||
}, [])
|
||||
|
||||
const onClickPreview = useCallback(() => {
|
||||
isPreview.current = true
|
||||
formRef.current?.submit()
|
||||
}, [])
|
||||
|
||||
const handleSubmit = useCallback((data: Record<string, any>) => {
|
||||
isPreview.current ? handlePreviewChunks(data) : handleProcess(data)
|
||||
}, [handlePreviewChunks, handleProcess])
|
||||
|
||||
const handlePreviewFileChange = useCallback((file: DocumentItem) => {
|
||||
onClickPreview()
|
||||
}, [onClickPreview])
|
||||
|
||||
const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
|
||||
onClickPreview()
|
||||
}, [onClickPreview])
|
||||
|
||||
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
|
||||
onClickPreview()
|
||||
}, [onClickPreview])
|
||||
|
||||
if (isFetchingDocumentDetail) {
|
||||
return (
|
||||
<Loading type='app' />
|
||||
)
|
||||
}
|
||||
|
||||
if (error)
|
||||
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
|
||||
|
||||
return (
|
||||
<div
|
||||
className='relative flex h-[calc(100vh-56px)] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
|
||||
>
|
||||
<div className='flex h-full flex-1 flex-col px-14'>
|
||||
<LeftHeader title={t('datasetPipeline.documentSettings.title')} />
|
||||
<div className='grow overflow-y-auto'>
|
||||
<ProcessDocuments
|
||||
ref={formRef}
|
||||
documentId={documentId}
|
||||
onProcess={onClickProcess}
|
||||
onPreview={onClickPreview}
|
||||
onSubmit={handleSubmit}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{/* Preview */}
|
||||
<div className='flex h-full flex-1 shrink-0 pl-2 pt-2'>
|
||||
<ChunkPreview
|
||||
dataSourceType={documentDetail!.data_source_type as DatasourceType}
|
||||
// @ts-expect-error mock data // todo: remove mock data
|
||||
files={[{
|
||||
id: '12345678',
|
||||
name: 'test-file',
|
||||
extension: 'txt',
|
||||
}]}
|
||||
onlineDocuments={[]}
|
||||
websitePages={[]}
|
||||
isIdle={true}
|
||||
isPending={true}
|
||||
estimateData={estimateData}
|
||||
onPreview={onClickPreview}
|
||||
handlePreviewFileChange={handlePreviewFileChange}
|
||||
handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
|
||||
handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default PipelineSettings
|
||||
@ -0,0 +1,42 @@
|
||||
import React, { useCallback } from 'react'
|
||||
import { RiArrowLeftLine } from '@remixicon/react'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useRouter } from 'next/navigation'
|
||||
import Effect from '@/app/components/base/effect'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type LeftHeaderProps = {
|
||||
title: string
|
||||
}
|
||||
|
||||
const LeftHeader = ({
|
||||
title,
|
||||
}: LeftHeaderProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { back } = useRouter()
|
||||
|
||||
const navigateBack = useCallback(() => {
|
||||
back()
|
||||
}, [back])
|
||||
|
||||
return (
|
||||
<div className='relative flex flex-col gap-y-0.5 pb-2 pt-4'>
|
||||
<div className='system-2xs-semibold-uppercase bg-pipeline-add-documents-title-bg bg-clip-text text-transparent'>
|
||||
{title}
|
||||
</div>
|
||||
<div className='system-md-semibold text-text-primary'>
|
||||
{t('datasetPipeline.addDocuments.steps.processDocuments')}
|
||||
</div>
|
||||
<Button
|
||||
variant='secondary-accent'
|
||||
className='absolute -left-11 top-3.5 size-9 rounded-full p-0'
|
||||
onClick={navigateBack}
|
||||
>
|
||||
<RiArrowLeftLine className='size-5 ' />
|
||||
</Button>
|
||||
<Effect className='left-8 top-[-34px] opacity-20' />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(LeftHeader)
|
||||
@ -0,0 +1,26 @@
|
||||
import React from 'react'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type ActionsProps = {
|
||||
onProcess: () => void
|
||||
}
|
||||
|
||||
const Actions = ({
|
||||
onProcess,
|
||||
}: ActionsProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex items-center justify-end'>
|
||||
<Button
|
||||
variant='primary'
|
||||
onClick={onProcess}
|
||||
>
|
||||
{t('datasetPipeline.operations.saveAndProcess')}
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Actions)
|
||||
@ -0,0 +1,11 @@
|
||||
import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
|
||||
|
||||
export const useConfigurations = (documentdId: string) => {
|
||||
const initialData: Record<string, any> = {}
|
||||
const configurations: BaseConfiguration[] = []
|
||||
|
||||
return {
|
||||
initialData,
|
||||
configurations,
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
|
||||
import { useConfigurations } from './hooks'
|
||||
import Actions from './actions'
|
||||
import Form from '../../../../create-from-pipeline/process-documents/form'
|
||||
|
||||
type ProcessDocumentsProps = {
|
||||
documentId: string
|
||||
ref: React.RefObject<any>
|
||||
onProcess: () => void
|
||||
onPreview: () => void
|
||||
onSubmit: (data: Record<string, any>) => void
|
||||
}
|
||||
|
||||
const ProcessDocuments = ({
|
||||
documentId,
|
||||
onProcess,
|
||||
onPreview,
|
||||
onSubmit,
|
||||
ref,
|
||||
}: ProcessDocumentsProps) => {
|
||||
const { initialData, configurations } = useConfigurations(documentId)
|
||||
const schema = generateZodSchema(configurations)
|
||||
|
||||
return (
|
||||
<div className='flex flex-col gap-y-4 pt-4'>
|
||||
<Form
|
||||
ref={ref}
|
||||
initialData={initialData}
|
||||
configurations={configurations}
|
||||
schema={schema}
|
||||
onSubmit={onSubmit}
|
||||
onPreview={onPreview}
|
||||
/>
|
||||
<Actions onProcess={onProcess} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default ProcessDocuments
|
||||
Loading…
Reference in New Issue