ngskcloud
/
gcgj-dify-1.7.0
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: step 2

Browse Source
pull/12097/head
AkaraChen 1 year ago
parent dfdc4ed3b1
commit bebad5cbdd
2 changed files with 150 additions and 111 deletions
Show Stats Download Patch File Download Diff File

170
web/app/components/datasets/create/step-two/index.tsx
Unescape Escape View File

@ -25,11 +25,7 @@ import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
import PreviewItem, { PreviewType } from './preview-item'
import cn from '@/utils/classnames'
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
createDocument,
createFirstDocument,
fetchDefaultProcessRule,
} from '@/service/datasets'
import Button from '@/app/components/base/button'
import FloatRightContainer from '@/app/components/base/float-right-container'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
@ -55,7 +51,7 @@ import { MessageChatSquare } from '@/app/components/base/icons/src/public/common
import { IS_CE_EDITION } from '@/config'
import Switch from '@/app/components/base/switch'
import Divider from '@/app/components/base/divider'
import { getNotionInfo, getWebsiteInfo, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/use-datasets'
import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/use-datasets'
import Loading from '@/app/components/base/loading'
const TextLabel: FC<PropsWithChildren> = (props) => {
@ -174,8 +170,7 @@ const StepTwo = ({
(datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'),
)
const [QATipHide, setQATipHide] = useState(false)
const [previewSwitched, setPreviewSwitched] = useState(false)
const [isCreating, setIsCreating] = useState(false)
const [qaPreviewSwitched, setQAPreviewSwitched] = useState(false)
const [parentChildConfig, setParentChildConfig] = useState<ParentChildConfig>(defaultParentChildConfig)
@ -250,21 +245,21 @@ const StepTwo = ({
? notionIndexingEstimateQuery.data
: websiteIndexingEstimateQuery.data
const getIsEstimateReady = useCallback(() => {
if (dataSourceType === DataSourceType.FILE)
return fileIndexingEstimateQuery.isSuccess
// const getIsEstimateReady = useCallback(() => {
// if (dataSourceType === DataSourceType.FILE)
// return fileIndexingEstimateQuery.isSuccess
if (dataSourceType === DataSourceType.NOTION)
return notionIndexingEstimateQuery.isSuccess
// if (dataSourceType === DataSourceType.NOTION)
// return notionIndexingEstimateQuery.isSuccess
if (dataSourceType === DataSourceType.WEB)
return websiteIndexingEstimateQuery.isSuccess
}, [dataSourceType, fileIndexingEstimateQuery.isSuccess, notionIndexingEstimateQuery.isSuccess, websiteIndexingEstimateQuery.isSuccess])
// if (dataSourceType === DataSourceType.WEB)
// return websiteIndexingEstimateQuery.isSuccess
// }, [dataSourceType, fileIndexingEstimateQuery.isSuccess, notionIndexingEstimateQuery.isSuccess, websiteIndexingEstimateQuery.isSuccess])
const getFileName = (name: string) => {
const arr = name.split('.')
return arr.slice(0, -1).join('.')
}
// const getFileName = (name: string) => {
// const arr = name.split('.')
// return arr.slice(0, -1).join('.')
// }
const getRuleName = (key: string) => {
if (key === 'remove_extra_spaces')
@ -304,7 +299,7 @@ const StepTwo = ({
return
}
fetchEstimate()
setPreviewSwitched(false)
setQAPreviewSwitched(false)
}
const {
@ -403,20 +398,22 @@ const StepTwo = ({
return params
}
const getRules = async () => {
try {
const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' })
const separator = res.rules.segmentation.separator
const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({
onSuccess(data) {
const separator = data.rules.segmentation.separator
setSegmentIdentifier(separator)
setMax(res.rules.segmentation.max_tokens)
setOverlap(res.rules.segmentation.chunk_overlap!)
setRules(res.rules.pre_processing_rules)
setDefaultConfig(res.rules)
}
catch (err) {
console.log(err)
}
}
setMax(data.rules.segmentation.max_tokens)
setOverlap(data.rules.segmentation.chunk_overlap!)
setRules(data.rules.pre_processing_rules)
setDefaultConfig(data.rules)
},
onError(error) {
Toast.notify({
type: 'error',
message: `${error}`,
})
},
})
const getRulesFromDetail = () => {
if (documentDetail) {
@ -426,7 +423,7 @@ const StepTwo = ({
const overlap = rules.segmentation.chunk_overlap
setSegmentIdentifier(separator)
setMax(max)
setOverlap(overlap)
setOverlap(overlap as number)
setRules(rules.pre_processing_rules)
setDefaultConfig(rules)
}
@ -437,48 +434,55 @@ const StepTwo = ({
setSegmentationType(documentDetail.dataset_process_rule.mode)
}
const createHandle = async () => {
if (isCreating)
return
setIsCreating(true)
try {
let res
const params = getCreationParams()
if (!params)
return false
setIsCreating(true)
if (!datasetId) {
res = await createFirstDocument({
body: params as CreateDocumentReq,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(res)
// eslint-disable-next-line @typescript-eslint/no-use-before-define
updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string)
}
else {
res = await createDocument({
datasetId,
body: params as CreateDocumentReq,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(res)
}
if (mutateDatasetRes)
mutateDatasetRes()
onStepChange && onStepChange(+1)
isSetting && onSave && onSave()
}
catch (err) {
const createFirstDocumentMutation = useCreateFirstDocument({
onError(error) {
Toast.notify({
type: 'error',
message: `${error}`,
})
},
})
const createDocumentMutation = useCreateDocument(datasetId!, {
onError(error) {
Toast.notify({
type: 'error',
message: `${err}`,
message: `${error}`,
})
},
})
const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
const createHandle = async () => {
const params = getCreationParams()
if (!params)
return false
if (!datasetId) {
await createFirstDocumentMutation.mutateAsync(
params,
{
onSuccess(data) {
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(data)
// eslint-disable-next-line @typescript-eslint/no-use-before-define
updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string)
},
},
)
}
finally {
setIsCreating(false)
else {
await createDocumentMutation.mutateAsync(params, {
onSuccess(data) {
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(data)
},
})
}
if (mutateDatasetRes)
mutateDatasetRes()
onStepChange && onStepChange(+1)
isSetting && onSave && onSave()
}
const handleDocformSwitch = (isQAMode: boolean) => {
@ -488,8 +492,8 @@ const StepTwo = ({
setDocForm(DocForm.TEXT)
}
const previewSwitch = async (language?: string) => {
setPreviewSwitched(true)
const previewSwitch = () => {
setQAPreviewSwitched(true)
setIsLanguageSelectDisabled(true)
fetchEstimate()
}
@ -497,8 +501,8 @@ const StepTwo = ({
const handleSelect = (language: string) => {
setDocLanguage(language)
// Switch language, re-cutter
if (docForm === DocForm.QA && previewSwitched)
previewSwitch(language)
if (docForm === DocForm.QA && qaPreviewSwitched)
previewSwitch()
}
const changeToEconomicalType = () => {
@ -511,7 +515,7 @@ const StepTwo = ({
useEffect(() => {
// fetch rules
if (!isSetting) {
getRules()
fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule')
}
else {
getRulesFromDetail()
@ -909,12 +913,12 @@ const StepTwo = ({
<div className='flex items-center justify-between px-8'>
<div className='grow flex items-center'>
<div>{t('datasetCreation.stepTwo.previewTitle')}</div>
{docForm === DocForm.QA && !previewSwitched && (
{docForm === DocForm.QA && !qaPreviewSwitched && (
<Button className='ml-2' variant='secondary-accent' onClick={() => previewSwitch()}>{t('datasetCreation.stepTwo.previewButton')}</Button>
)}
</div>
</div>
{docForm === DocForm.QA && !previewSwitched && (
{docForm === DocForm.QA && !qaPreviewSwitched && (
<div className='px-8 pr-12 text-xs text-gray-500'>
<span>{t('datasetCreation.stepTwo.previewSwitchTipStart')}</span>
<span className='text-amber-600'>{t('datasetCreation.stepTwo.previewSwitchTipEnd')}</span>
@ -922,26 +926,26 @@ const StepTwo = ({
)}
</div>
<div className='my-4 px-8 space-y-4'>
{previewSwitched && docForm === DocForm.QA && estimate?.qa_preview && (
{qaPreviewSwitched && docForm === DocForm.QA && estimate?.qa_preview && (
<>
{estimate?.qa_preview.map((item, index) => (
<PreviewItem type={PreviewType.QA} key={item.question} qa={item} index={index + 1} />
))}
</>
)}
{(docForm === DocForm.TEXT || !previewSwitched) && estimate?.preview && (
{(docForm === DocForm.TEXT || !qaPreviewSwitched) && estimate?.preview && (
<>
{estimate?.preview.map((item, index) => (
<PreviewItem type={PreviewType.TEXT} key={item} content={item} index={index + 1} />
))}
</>
)}
{previewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && (
{qaPreviewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && (
<div className='flex items-center justify-center h-[200px]'>
<Loading type='area' />
</div>
)}
{!previewSwitched && !estimate?.preview && (
{!qaPreviewSwitched && !estimate?.preview && (
<div className='flex items-center justify-center h-[200px]'>
<Loading type='area' />
</div>

91
web/service/use-datasets.ts
Unescape Escape View File

@ -1,9 +1,9 @@
import groupBy from 'lodash-es/groupBy'
import type { MutationOptions } from '@tanstack/react-query'
import { useMutation } from '@tanstack/react-query'
import { fetchFileIndexingEstimate } from './datasets'
import { createDocument, createFirstDocument, fetchDefaultProcessRule, fetchFileIndexingEstimate } from './datasets'
import { type IndexingType } from '@/app/components/datasets/create/step-two'
import type { CrawlOptions, CrawlResultItem, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets'
import type { DataSourceProvider, NotionPage } from '@/models/common'
export const getNotionInfo = (
@ -62,19 +62,6 @@ type GetFileIndexingEstimateParamsOptionFile = GetFileIndexingEstimateParamsOpti
files: CustomFile[]
}
type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.NOTION
notionPages: NotionPage[]
}
type GetFileIndexingEstimateParamsOptionWeb = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.WEB
websitePages: CrawlResultItem[]
crawlOptions?: CrawlOptions
websiteCrawlProvider: DataSourceProvider
websiteCrawlJobId: string
}
const getFileIndexingEstimateParamsForFile = ({
docForm,
docLanguage,
@ -99,6 +86,23 @@ const getFileIndexingEstimateParamsForFile = ({
}
}
export const useFetchFileIndexingEstimateForFile = (
options: GetFileIndexingEstimateParamsOptionFile,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForFile(options))
},
...mutationOptions,
})
}
type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.NOTION
notionPages: NotionPage[]
}
const getFileIndexingEstimateParamsForNotion = ({
docForm,
docLanguage,
@ -121,6 +125,26 @@ const getFileIndexingEstimateParamsForNotion = ({
}
}
export const useFetchFileIndexingEstimateForNotion = (
options: GetFileIndexingEstimateParamsOptionNotion,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForNotion(options))
},
...mutationOptions,
})
}
type GetFileIndexingEstimateParamsOptionWeb = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.WEB
websitePages: CrawlResultItem[]
crawlOptions?: CrawlOptions
websiteCrawlProvider: DataSourceProvider
websiteCrawlJobId: string
}
const getFileIndexingEstimateParamsForWeb = ({
docForm,
docLanguage,
@ -151,37 +175,48 @@ const getFileIndexingEstimateParamsForWeb = ({
}
}
export const useFetchFileIndexingEstimateForFile = (
options: GetFileIndexingEstimateParamsOptionFile,
export const useFetchFileIndexingEstimateForWeb = (
options: GetFileIndexingEstimateParamsOptionWeb,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForFile(options))
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForWeb(options))
},
...mutationOptions,
})
}
export const useFetchFileIndexingEstimateForNotion = (
options: GetFileIndexingEstimateParamsOptionNotion,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
export const useCreateFirstDocument = (
mutationOptions: MutationOptions<createDocumentResponse, Error, CreateDocumentReq> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForNotion(options))
mutationFn: async (createDocumentReq: CreateDocumentReq,
) => {
return createFirstDocument({ body: createDocumentReq })
},
...mutationOptions,
})
}
export const useFetchFileIndexingEstimateForWeb = (
options: GetFileIndexingEstimateParamsOptionWeb,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
export const useCreateDocument = (
datasetId: string,
mutationOptions: MutationOptions<createDocumentResponse, Error, CreateDocumentReq> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForWeb(options))
mutationFn: async (req: CreateDocumentReq) => {
return createDocument({ datasetId, body: req })
},
...mutationOptions,
})
}
export const useFetchDefaultProcessRule = (
mutationOptions: MutationOptions<ProcessRuleResponse, Error, string> = {},
) => {
return useMutation({
mutationFn: async (url: string) => {
return fetchDefaultProcessRule({ url })
},
...mutationOptions,
})

Write Preview
Loading…
Cancel
Save