ngskcloud
/
gcgj-dify-1.7.0
17
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: step 2

Browse Source
pull/12097/head
AkaraChen 1 year ago
parent dfdc4ed3b1
commit bebad5cbdd
2 changed files with 150 additions and 111 deletions
Show Stats Download Patch File Download Diff File

146
web/app/components/datasets/create/step-two/index.tsx
Unescape Escape View File

@ -25,11 +25,7 @@ import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
import PreviewItem, { PreviewType } from './preview-item' import PreviewItem, { PreviewType } from './preview-item'
import cn from '@/utils/classnames' import cn from '@/utils/classnames'
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
createDocument,
createFirstDocument,
fetchDefaultProcessRule,
} from '@/service/datasets'
import Button from '@/app/components/base/button' import Button from '@/app/components/base/button'
import FloatRightContainer from '@/app/components/base/float-right-container' import FloatRightContainer from '@/app/components/base/float-right-container'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
@ -55,7 +51,7 @@ import { MessageChatSquare } from '@/app/components/base/icons/src/public/common
import { IS_CE_EDITION } from '@/config' import { IS_CE_EDITION } from '@/config'
import Switch from '@/app/components/base/switch' import Switch from '@/app/components/base/switch'
import Divider from '@/app/components/base/divider' import Divider from '@/app/components/base/divider'
import { getNotionInfo, getWebsiteInfo, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/use-datasets' import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/use-datasets'
import Loading from '@/app/components/base/loading' import Loading from '@/app/components/base/loading'
const TextLabel: FC<PropsWithChildren> = (props) => { const TextLabel: FC<PropsWithChildren> = (props) => {
@ -174,8 +170,7 @@ const StepTwo = ({
(datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'), (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'),
) )
const [QATipHide, setQATipHide] = useState(false) const [QATipHide, setQATipHide] = useState(false)
const [previewSwitched, setPreviewSwitched] = useState(false) const [qaPreviewSwitched, setQAPreviewSwitched] = useState(false)
const [isCreating, setIsCreating] = useState(false)
const [parentChildConfig, setParentChildConfig] = useState<ParentChildConfig>(defaultParentChildConfig) const [parentChildConfig, setParentChildConfig] = useState<ParentChildConfig>(defaultParentChildConfig)
@ -250,21 +245,21 @@ const StepTwo = ({
? notionIndexingEstimateQuery.data ? notionIndexingEstimateQuery.data
: websiteIndexingEstimateQuery.data : websiteIndexingEstimateQuery.data
const getIsEstimateReady = useCallback(() => { // const getIsEstimateReady = useCallback(() => {
if (dataSourceType === DataSourceType.FILE) // if (dataSourceType === DataSourceType.FILE)
return fileIndexingEstimateQuery.isSuccess // return fileIndexingEstimateQuery.isSuccess
if (dataSourceType === DataSourceType.NOTION) // if (dataSourceType === DataSourceType.NOTION)
return notionIndexingEstimateQuery.isSuccess // return notionIndexingEstimateQuery.isSuccess
if (dataSourceType === DataSourceType.WEB) // if (dataSourceType === DataSourceType.WEB)
return websiteIndexingEstimateQuery.isSuccess // return websiteIndexingEstimateQuery.isSuccess
}, [dataSourceType, fileIndexingEstimateQuery.isSuccess, notionIndexingEstimateQuery.isSuccess, websiteIndexingEstimateQuery.isSuccess]) // }, [dataSourceType, fileIndexingEstimateQuery.isSuccess, notionIndexingEstimateQuery.isSuccess, websiteIndexingEstimateQuery.isSuccess])
const getFileName = (name: string) => { // const getFileName = (name: string) => {
const arr = name.split('.') // const arr = name.split('.')
return arr.slice(0, -1).join('.') // return arr.slice(0, -1).join('.')
} // }
const getRuleName = (key: string) => { const getRuleName = (key: string) => {
if (key === 'remove_extra_spaces') if (key === 'remove_extra_spaces')
@ -304,7 +299,7 @@ const StepTwo = ({
return return
} }
fetchEstimate() fetchEstimate()
setPreviewSwitched(false) setQAPreviewSwitched(false)
} }
const { const {
@ -403,20 +398,22 @@ const StepTwo = ({
return params return params
} }
const getRules = async () => { const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({
try { onSuccess(data) {
const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' }) const separator = data.rules.segmentation.separator
const separator = res.rules.segmentation.separator
setSegmentIdentifier(separator) setSegmentIdentifier(separator)
setMax(res.rules.segmentation.max_tokens) setMax(data.rules.segmentation.max_tokens)
setOverlap(res.rules.segmentation.chunk_overlap!) setOverlap(data.rules.segmentation.chunk_overlap!)
setRules(res.rules.pre_processing_rules) setRules(data.rules.pre_processing_rules)
setDefaultConfig(res.rules) setDefaultConfig(data.rules)
} },
catch (err) { onError(error) {
console.log(err) Toast.notify({
} type: 'error',
} message: `${error}`,
})
},
})
const getRulesFromDetail = () => { const getRulesFromDetail = () => {
if (documentDetail) { if (documentDetail) {
@ -426,7 +423,7 @@ const StepTwo = ({
const overlap = rules.segmentation.chunk_overlap const overlap = rules.segmentation.chunk_overlap
setSegmentIdentifier(separator) setSegmentIdentifier(separator)
setMax(max) setMax(max)
setOverlap(overlap) setOverlap(overlap as number)
setRules(rules.pre_processing_rules) setRules(rules.pre_processing_rules)
setDefaultConfig(rules) setDefaultConfig(rules)
} }
@ -437,49 +434,56 @@ const StepTwo = ({
setSegmentationType(documentDetail.dataset_process_rule.mode) setSegmentationType(documentDetail.dataset_process_rule.mode)
} }
const createFirstDocumentMutation = useCreateFirstDocument({
onError(error) {
Toast.notify({
type: 'error',
message: `${error}`,
})
},
})
const createDocumentMutation = useCreateDocument(datasetId!, {
onError(error) {
Toast.notify({
type: 'error',
message: `${error}`,
})
},
})
const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
const createHandle = async () => { const createHandle = async () => {
if (isCreating)
return
setIsCreating(true)
try {
let res
const params = getCreationParams() const params = getCreationParams()
if (!params) if (!params)
return false return false
setIsCreating(true)
if (!datasetId) { if (!datasetId) {
res = await createFirstDocument({ await createFirstDocumentMutation.mutateAsync(
body: params as CreateDocumentReq, params,
}) {
onSuccess(data) {
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(res) updateResultCache && updateResultCache(data)
// eslint-disable-next-line @typescript-eslint/no-use-before-define // eslint-disable-next-line @typescript-eslint/no-use-before-define
updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string) updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string)
},
},
)
} }
else { else {
res = await createDocument({ await createDocumentMutation.mutateAsync(params, {
datasetId, onSuccess(data) {
body: params as CreateDocumentReq,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(res) updateResultCache && updateResultCache(data)
},
})
} }
if (mutateDatasetRes) if (mutateDatasetRes)
mutateDatasetRes() mutateDatasetRes()
onStepChange && onStepChange(+1) onStepChange && onStepChange(+1)
isSetting && onSave && onSave() isSetting && onSave && onSave()
} }
catch (err) {
Toast.notify({
type: 'error',
message: `${err}`,
})
}
finally {
setIsCreating(false)
}
}
const handleDocformSwitch = (isQAMode: boolean) => { const handleDocformSwitch = (isQAMode: boolean) => {
if (isQAMode) if (isQAMode)
@ -488,8 +492,8 @@ const StepTwo = ({
setDocForm(DocForm.TEXT) setDocForm(DocForm.TEXT)
} }
const previewSwitch = async (language?: string) => { const previewSwitch = () => {
setPreviewSwitched(true) setQAPreviewSwitched(true)
setIsLanguageSelectDisabled(true) setIsLanguageSelectDisabled(true)
fetchEstimate() fetchEstimate()
} }
@ -497,8 +501,8 @@ const StepTwo = ({
const handleSelect = (language: string) => { const handleSelect = (language: string) => {
setDocLanguage(language) setDocLanguage(language)
// Switch language, re-cutter // Switch language, re-cutter
if (docForm === DocForm.QA && previewSwitched) if (docForm === DocForm.QA && qaPreviewSwitched)
previewSwitch(language) previewSwitch()
} }
const changeToEconomicalType = () => { const changeToEconomicalType = () => {
@ -511,7 +515,7 @@ const StepTwo = ({
useEffect(() => { useEffect(() => {
// fetch rules // fetch rules
if (!isSetting) { if (!isSetting) {
getRules() fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule')
} }
else { else {
getRulesFromDetail() getRulesFromDetail()
@ -909,12 +913,12 @@ const StepTwo = ({
<div className='flex items-center justify-between px-8'> <div className='flex items-center justify-between px-8'>
<div className='grow flex items-center'> <div className='grow flex items-center'>
<div>{t('datasetCreation.stepTwo.previewTitle')}</div> <div>{t('datasetCreation.stepTwo.previewTitle')}</div>
{docForm === DocForm.QA && !previewSwitched && ( {docForm === DocForm.QA && !qaPreviewSwitched && (
<Button className='ml-2' variant='secondary-accent' onClick={() => previewSwitch()}>{t('datasetCreation.stepTwo.previewButton')}</Button> <Button className='ml-2' variant='secondary-accent' onClick={() => previewSwitch()}>{t('datasetCreation.stepTwo.previewButton')}</Button>
)} )}
</div> </div>
</div> </div>
{docForm === DocForm.QA && !previewSwitched && ( {docForm === DocForm.QA && !qaPreviewSwitched && (
<div className='px-8 pr-12 text-xs text-gray-500'> <div className='px-8 pr-12 text-xs text-gray-500'>
<span>{t('datasetCreation.stepTwo.previewSwitchTipStart')}</span> <span>{t('datasetCreation.stepTwo.previewSwitchTipStart')}</span>
<span className='text-amber-600'>{t('datasetCreation.stepTwo.previewSwitchTipEnd')}</span> <span className='text-amber-600'>{t('datasetCreation.stepTwo.previewSwitchTipEnd')}</span>
@ -922,26 +926,26 @@ const StepTwo = ({
)} )}
</div> </div>
<div className='my-4 px-8 space-y-4'> <div className='my-4 px-8 space-y-4'>
{previewSwitched && docForm === DocForm.QA && estimate?.qa_preview && ( {qaPreviewSwitched && docForm === DocForm.QA && estimate?.qa_preview && (
<> <>
{estimate?.qa_preview.map((item, index) => ( {estimate?.qa_preview.map((item, index) => (
<PreviewItem type={PreviewType.QA} key={item.question} qa={item} index={index + 1} /> <PreviewItem type={PreviewType.QA} key={item.question} qa={item} index={index + 1} />
))} ))}
</> </>
)} )}
{(docForm === DocForm.TEXT || !previewSwitched) && estimate?.preview && ( {(docForm === DocForm.TEXT || !qaPreviewSwitched) && estimate?.preview && (
<> <>
{estimate?.preview.map((item, index) => ( {estimate?.preview.map((item, index) => (
<PreviewItem type={PreviewType.TEXT} key={item} content={item} index={index + 1} /> <PreviewItem type={PreviewType.TEXT} key={item} content={item} index={index + 1} />
))} ))}
</> </>
)} )}
{previewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && ( {qaPreviewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && (
<div className='flex items-center justify-center h-[200px]'> <div className='flex items-center justify-center h-[200px]'>
<Loading type='area' /> <Loading type='area' />
</div> </div>
)} )}
{!previewSwitched && !estimate?.preview && ( {!qaPreviewSwitched && !estimate?.preview && (
<div className='flex items-center justify-center h-[200px]'> <div className='flex items-center justify-center h-[200px]'>
<Loading type='area' /> <Loading type='area' />
</div> </div>

91
web/service/use-datasets.ts
Unescape Escape View File

@ -1,9 +1,9 @@
import groupBy from 'lodash-es/groupBy' import groupBy from 'lodash-es/groupBy'
import type { MutationOptions } from '@tanstack/react-query' import type { MutationOptions } from '@tanstack/react-query'
import { useMutation } from '@tanstack/react-query' import { useMutation } from '@tanstack/react-query'
import { fetchFileIndexingEstimate } from './datasets' import { createDocument, createFirstDocument, fetchDefaultProcessRule, fetchFileIndexingEstimate } from './datasets'
import { type IndexingType } from '@/app/components/datasets/create/step-two' import { type IndexingType } from '@/app/components/datasets/create/step-two'
import type { CrawlOptions, CrawlResultItem, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule } from '@/models/datasets' import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets'
import type { DataSourceProvider, NotionPage } from '@/models/common' import type { DataSourceProvider, NotionPage } from '@/models/common'
export const getNotionInfo = ( export const getNotionInfo = (
@ -62,19 +62,6 @@ type GetFileIndexingEstimateParamsOptionFile = GetFileIndexingEstimateParamsOpti
files: CustomFile[] files: CustomFile[]
} }
type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.NOTION
notionPages: NotionPage[]
}
type GetFileIndexingEstimateParamsOptionWeb = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.WEB
websitePages: CrawlResultItem[]
crawlOptions?: CrawlOptions
websiteCrawlProvider: DataSourceProvider
websiteCrawlJobId: string
}
const getFileIndexingEstimateParamsForFile = ({ const getFileIndexingEstimateParamsForFile = ({
docForm, docForm,
docLanguage, docLanguage,
@ -99,6 +86,23 @@ const getFileIndexingEstimateParamsForFile = ({
} }
} }
export const useFetchFileIndexingEstimateForFile = (
options: GetFileIndexingEstimateParamsOptionFile,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForFile(options))
},
...mutationOptions,
})
}
type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.NOTION
notionPages: NotionPage[]
}
const getFileIndexingEstimateParamsForNotion = ({ const getFileIndexingEstimateParamsForNotion = ({
docForm, docForm,
docLanguage, docLanguage,
@ -121,6 +125,26 @@ const getFileIndexingEstimateParamsForNotion = ({
} }
} }
export const useFetchFileIndexingEstimateForNotion = (
options: GetFileIndexingEstimateParamsOptionNotion,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => {
return useMutation({
mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForNotion(options))
},
...mutationOptions,
})
}
type GetFileIndexingEstimateParamsOptionWeb = GetFileIndexingEstimateParamsOptionBase & {
dataSourceType: DataSourceType.WEB
websitePages: CrawlResultItem[]
crawlOptions?: CrawlOptions
websiteCrawlProvider: DataSourceProvider
websiteCrawlJobId: string
}
const getFileIndexingEstimateParamsForWeb = ({ const getFileIndexingEstimateParamsForWeb = ({
docForm, docForm,
docLanguage, docLanguage,
@ -151,37 +175,48 @@ const getFileIndexingEstimateParamsForWeb = ({
} }
} }
export const useFetchFileIndexingEstimateForFile = ( export const useFetchFileIndexingEstimateForWeb = (
options: GetFileIndexingEstimateParamsOptionFile, options: GetFileIndexingEstimateParamsOptionWeb,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {}, mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => { ) => {
return useMutation({ return useMutation({
mutationFn: async () => { mutationFn: async () => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForFile(options)) return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForWeb(options))
}, },
...mutationOptions, ...mutationOptions,
}) })
} }
export const useFetchFileIndexingEstimateForNotion = ( export const useCreateFirstDocument = (
options: GetFileIndexingEstimateParamsOptionNotion, mutationOptions: MutationOptions<createDocumentResponse, Error, CreateDocumentReq> = {},
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {},
) => { ) => {
return useMutation({ return useMutation({
mutationFn: async () => { mutationFn: async (createDocumentReq: CreateDocumentReq,
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForNotion(options)) ) => {
return createFirstDocument({ body: createDocumentReq })
}, },
...mutationOptions, ...mutationOptions,
}) })
} }
export const useFetchFileIndexingEstimateForWeb = ( export const useCreateDocument = (
options: GetFileIndexingEstimateParamsOptionWeb, datasetId: string,
mutationOptions: MutationOptions<FileIndexingEstimateResponse> = {}, mutationOptions: MutationOptions<createDocumentResponse, Error, CreateDocumentReq> = {},
) => { ) => {
return useMutation({ return useMutation({
mutationFn: async () => { mutationFn: async (req: CreateDocumentReq) => {
return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForWeb(options)) return createDocument({ datasetId, body: req })
},
...mutationOptions,
})
}
export const useFetchDefaultProcessRule = (
mutationOptions: MutationOptions<ProcessRuleResponse, Error, string> = {},
) => {
return useMutation({
mutationFn: async (url: string) => {
return fetchDefaultProcessRule({ url })
}, },
...mutationOptions, ...mutationOptions,
}) })

Write Preview
Loading…
Cancel
Save