feat: knowledge base node
zxhlyh
11 months ago
parent
12c060b795
commit
a478d95950
@ -1,20 +0,0 @@
|
||||
import { FullTextSearch } from '@/app/components/base/icons/src/vender/knowledge'
|
||||
import OptionCard from '../option-card'
|
||||
|
||||
const FullTextSearchCard = () => {
|
||||
return (
|
||||
<OptionCard
|
||||
icon={<FullTextSearch className='h-[15px] w-[15px] text-text-tertiary' />}
|
||||
title='Full-Text Search'
|
||||
description="Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user's query. Users can choose to set weights or configure to a Rerank model."
|
||||
effectColor='purple'
|
||||
>
|
||||
<div className='flex flex-col gap-2'>
|
||||
<div>Vector Search Settings</div>
|
||||
<div>Additional Settings</div>
|
||||
</div>
|
||||
</OptionCard>
|
||||
)
|
||||
}
|
||||
|
||||
export default FullTextSearchCard
|
||||
@ -0,0 +1,65 @@
|
||||
import {
|
||||
FullTextSearch,
|
||||
HybridSearch,
|
||||
VectorSearch,
|
||||
} from '@/app/components/base/icons/src/vender/knowledge'
|
||||
import {
|
||||
HybridSearchModeEnum,
|
||||
RetrievalSearchMethodEnum,
|
||||
} from '../../types'
|
||||
import type {
|
||||
HybridSearchModeOption,
|
||||
Option,
|
||||
} from './type'
|
||||
|
||||
export const useRetrievalSetting = () => {
|
||||
const VectorSearchOption: Option = {
|
||||
id: RetrievalSearchMethodEnum.semantic,
|
||||
icon: VectorSearch as any,
|
||||
title: 'Vector Search',
|
||||
description: 'Generate query embeddings and search for the text chunk most similar to its vector representation.',
|
||||
effectColor: 'purple',
|
||||
}
|
||||
const FullTextSearchOption: Option = {
|
||||
id: RetrievalSearchMethodEnum.fullText,
|
||||
icon: FullTextSearch as any,
|
||||
title: 'Full-Text Search',
|
||||
description: 'Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user\'s query. Users can choose to set weights or configure to a Rerank model.',
|
||||
effectColor: 'purple',
|
||||
}
|
||||
const HybridSearchOption: Option = {
|
||||
id: RetrievalSearchMethodEnum.hybrid,
|
||||
icon: HybridSearch as any,
|
||||
title: 'Hybrid Search',
|
||||
description: 'Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user\'s query. Users can choose to set weights or configure to a Rerank model.',
|
||||
effectColor: 'purple',
|
||||
}
|
||||
|
||||
const options = [
|
||||
VectorSearchOption,
|
||||
FullTextSearchOption,
|
||||
HybridSearchOption,
|
||||
]
|
||||
|
||||
const WeightedScoreModeOption: HybridSearchModeOption = {
|
||||
id: HybridSearchModeEnum.WeightedScore,
|
||||
title: 'Weighted Score',
|
||||
description: 'By adjusting the weights assigned, this rerank strategy determines whether to prioritize semantic or keyword matching.',
|
||||
}
|
||||
|
||||
const RerankModelModeOption: HybridSearchModeOption = {
|
||||
id: HybridSearchModeEnum.RerankingModel,
|
||||
title: 'Rerank Model',
|
||||
description: 'Rerank model will reorder the candidate document list based on the semantic match with user query, improving the results of semantic ranking.',
|
||||
}
|
||||
|
||||
const hybridSearchModeOptions = [
|
||||
WeightedScoreModeOption,
|
||||
RerankModelModeOption,
|
||||
]
|
||||
|
||||
return {
|
||||
options,
|
||||
hybridSearchModeOptions,
|
||||
}
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
import { HybridSearch } from '@/app/components/base/icons/src/vender/knowledge'
|
||||
import OptionCard from '../option-card'
|
||||
|
||||
const HybridSearchCard = () => {
|
||||
return (
|
||||
<OptionCard
|
||||
icon={<HybridSearch className='h-[15px] w-[15px] text-text-tertiary' />}
|
||||
title='Hybrid Search'
|
||||
description="Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user's query. Users can choose to set weights or configure to a Rerank model."
|
||||
effectColor='purple'
|
||||
isRecommended
|
||||
>
|
||||
<div className='flex flex-col gap-2'>
|
||||
<div>Vector Search Settings</div>
|
||||
<div>Additional Settings</div>
|
||||
</div>
|
||||
</OptionCard>
|
||||
)
|
||||
}
|
||||
|
||||
export default HybridSearchCard
|
||||
@ -0,0 +1,48 @@
|
||||
import {
|
||||
memo,
|
||||
useMemo,
|
||||
} from 'react'
|
||||
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
|
||||
import { useModelListAndDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
|
||||
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { RerankingModel } from '../../types'
|
||||
|
||||
type RerankingModelSelectorProps = {
|
||||
rerankingModel?: RerankingModel
|
||||
onRerankingModelChange?: (model: RerankingModel) => void
|
||||
}
|
||||
const RerankingModelSelector = ({
|
||||
rerankingModel,
|
||||
onRerankingModelChange,
|
||||
}: RerankingModelSelectorProps) => {
|
||||
const {
|
||||
modelList: rerankModelList,
|
||||
} = useModelListAndDefaultModel(ModelTypeEnum.rerank)
|
||||
const rerankModel = useMemo(() => {
|
||||
if (!rerankingModel)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider_name: rerankingModel.reranking_provider_name,
|
||||
model_name: rerankingModel.reranking_model_name,
|
||||
}
|
||||
}, [rerankingModel])
|
||||
|
||||
const handleRerankingModelChange = (model: DefaultModel) => {
|
||||
onRerankingModelChange?.({
|
||||
reranking_provider_name: model.provider,
|
||||
reranking_model_name: model.model,
|
||||
})
|
||||
}
|
||||
|
||||
return (
|
||||
<ModelSelector
|
||||
defaultModel={rerankModel && { provider: rerankModel.provider_name, model: rerankModel.model_name }}
|
||||
modelList={rerankModelList}
|
||||
onSelect={handleRerankingModelChange}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export default memo(RerankingModelSelector)
|
||||
@ -0,0 +1,42 @@
|
||||
import { memo } from 'react'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
import Input from '@/app/components/base/input'
|
||||
import Switch from '@/app/components/base/switch'
|
||||
|
||||
const TopKAndScoreThreshold = () => {
|
||||
return (
|
||||
<div className='grid grid-cols-2 gap-4'>
|
||||
<div>
|
||||
<div className='system-xs-medium mb-0.5 flex h-6 items-center text-text-secondary'>
|
||||
Top k
|
||||
<Tooltip
|
||||
triggerClassName='ml-0.5 shrink-0 w-3.5 h-3.5'
|
||||
popupContent='top k'
|
||||
/>
|
||||
</div>
|
||||
<Input
|
||||
type='number'
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<div className='mb-0.5 flex h-6 items-center'>
|
||||
<Switch
|
||||
className='mr-2'
|
||||
/>
|
||||
<div className='system-sm-medium grow truncate text-text-secondary'>
|
||||
Score Threshold
|
||||
</div>
|
||||
<Tooltip
|
||||
triggerClassName='shrink-0 ml-0.5 w-3.5 h-3.5'
|
||||
popupContent='Score Threshold'
|
||||
/>
|
||||
</div>
|
||||
<Input
|
||||
type='number'
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default memo(TopKAndScoreThreshold)
|
||||
@ -0,0 +1,20 @@
|
||||
import type { ComponentType } from 'react'
|
||||
import type {
|
||||
HybridSearchModeEnum,
|
||||
RetrievalSearchMethodEnum,
|
||||
} from '../../types'
|
||||
|
||||
export type Option = {
|
||||
id: RetrievalSearchMethodEnum
|
||||
icon: ComponentType<any>
|
||||
title: any
|
||||
description: string
|
||||
effectColor?: string
|
||||
showEffectColor?: boolean,
|
||||
}
|
||||
|
||||
export type HybridSearchModeOption = {
|
||||
id: HybridSearchModeEnum
|
||||
title: string
|
||||
description: string
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
import { VectorSearch } from '@/app/components/base/icons/src/vender/knowledge'
|
||||
import OptionCard from '../option-card'
|
||||
|
||||
const VectorSearchCard = () => {
|
||||
return (
|
||||
<OptionCard
|
||||
icon={<VectorSearch className='h-[15px] w-[15px] text-text-tertiary' />}
|
||||
title='Vector Search'
|
||||
description='Generate query embeddings and search for the text chunk most similar to its vector representation.'
|
||||
effectColor='purple'
|
||||
>
|
||||
<div className='flex flex-col gap-2'>
|
||||
<div>Vector Search Settings</div>
|
||||
<div>Additional Settings</div>
|
||||
</div>
|
||||
</OptionCard>
|
||||
)
|
||||
}
|
||||
|
||||
export default VectorSearchCard
|
||||
@ -1,19 +1,107 @@
|
||||
import {
|
||||
useCallback,
|
||||
useRef,
|
||||
} from 'react'
|
||||
import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud'
|
||||
import type { KnowledgeBaseNodeType } from '../types'
|
||||
|
||||
export const useConfig = (id: string, payload: KnowledgeBaseNodeType) => {
|
||||
const {
|
||||
inputs,
|
||||
setInputs,
|
||||
} = useNodeCrud(id, payload)
|
||||
const ref = useRef(inputs)
|
||||
|
||||
const handleInputsChange = useCallback((newInputs: KnowledgeBaseNodeType) => {
|
||||
setInputs(newInputs)
|
||||
ref.current = newInputs
|
||||
}, [setInputs, ref])
|
||||
import { useStoreApi } from 'reactflow'
|
||||
import { useNodeDataUpdate } from '@/app/components/workflow/hooks'
|
||||
import type {
|
||||
ChunkStructureEnum,
|
||||
HybridSearchModeEnum,
|
||||
IndexMethodEnum,
|
||||
KnowledgeBaseNodeType,
|
||||
RerankingModel,
|
||||
RetrievalSearchMethodEnum,
|
||||
} from '../types'
|
||||
|
||||
export const useConfig = (id: string) => {
|
||||
const store = useStoreApi()
|
||||
const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
|
||||
|
||||
const getNodeData = useCallback(() => {
|
||||
const { getNodes } = store.getState()
|
||||
const nodes = getNodes()
|
||||
|
||||
return nodes.find(node => node.id === id)
|
||||
}, [store, id])
|
||||
|
||||
const handleNodeDataUpdate = useCallback((data: Partial<KnowledgeBaseNodeType>) => {
|
||||
handleNodeDataUpdateWithSyncDraft({
|
||||
id,
|
||||
data,
|
||||
})
|
||||
}, [id, handleNodeDataUpdateWithSyncDraft])
|
||||
|
||||
const handleChunkStructureChange = useCallback((chunkStructure: ChunkStructureEnum) => {
|
||||
handleNodeDataUpdate({ chunk_structure: chunkStructure })
|
||||
}, [handleNodeDataUpdate])
|
||||
|
||||
const handleIndexMethodChange = useCallback((indexMethod: IndexMethodEnum) => {
|
||||
handleNodeDataUpdate({ indexing_technique: indexMethod })
|
||||
}, [handleNodeDataUpdate])
|
||||
|
||||
const handleKeywordNumberChange = useCallback((keywordNumber: number) => {
|
||||
handleNodeDataUpdate({ keyword_number: keywordNumber })
|
||||
}, [handleNodeDataUpdate])
|
||||
|
||||
const handleRetrievalSearchMethodChange = useCallback((searchMethod: RetrievalSearchMethodEnum) => {
|
||||
const nodeData = getNodeData()
|
||||
handleNodeDataUpdate({
|
||||
retrieval_model: {
|
||||
...nodeData?.data.retrieval_model,
|
||||
search_method: searchMethod,
|
||||
},
|
||||
})
|
||||
}, [getNodeData, handleNodeDataUpdate])
|
||||
|
||||
const handleHybridSearchModeChange = useCallback((hybridSearchMode: HybridSearchModeEnum) => {
|
||||
const nodeData = getNodeData()
|
||||
handleNodeDataUpdate({
|
||||
retrieval_model: {
|
||||
...nodeData?.data.retrieval_model,
|
||||
hybridSearchMode,
|
||||
},
|
||||
})
|
||||
}, [getNodeData, handleNodeDataUpdate])
|
||||
|
||||
const handleWeighedScoreChange = useCallback((weightedScore: { value: number[] }) => {
|
||||
const nodeData = getNodeData()
|
||||
handleNodeDataUpdate({
|
||||
retrieval_model: {
|
||||
...nodeData?.data.retrieval_model,
|
||||
weights: {
|
||||
weight_type: 'weighted_score',
|
||||
vector_setting: {
|
||||
vector_weight: weightedScore.value[0],
|
||||
embedding_provider_name: '',
|
||||
embedding_model_name: '',
|
||||
},
|
||||
keyword_setting: {
|
||||
keyword_weight: weightedScore.value[1],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}, [getNodeData, handleNodeDataUpdate])
|
||||
|
||||
const handleRerankingModelChange = useCallback((rerankingModel: RerankingModel) => {
|
||||
const nodeData = getNodeData()
|
||||
handleNodeDataUpdate({
|
||||
retrieval_model: {
|
||||
...nodeData?.data.retrieval_model,
|
||||
reranking_model: {
|
||||
reranking_provider_name: rerankingModel.reranking_provider_name,
|
||||
reranking_model_name: rerankingModel.reranking_model_name,
|
||||
},
|
||||
},
|
||||
})
|
||||
}, [getNodeData, handleNodeDataUpdate])
|
||||
|
||||
return {
|
||||
handleChunkStructureChange,
|
||||
handleIndexMethodChange,
|
||||
handleKeywordNumberChange,
|
||||
handleRetrievalSearchMethodChange,
|
||||
handleHybridSearchModeChange,
|
||||
handleWeighedScoreChange,
|
||||
handleRerankingModelChange,
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,3 +1,52 @@
|
||||
import type { CommonNodeType } from '@/app/components/workflow/types'
|
||||
import type { IndexingType } from '@/app/components/datasets/create/step-two'
|
||||
import type { RETRIEVE_METHOD } from '@/types/app'
|
||||
import type { WeightedScoreEnum } from '@/models/datasets'
|
||||
import type { RerankingModeEnum } from '@/models/datasets'
|
||||
export { WeightedScoreEnum } from '@/models/datasets'
|
||||
export { IndexingType as IndexMethodEnum } from '@/app/components/datasets/create/step-two'
|
||||
export { RETRIEVE_METHOD as RetrievalSearchMethodEnum } from '@/types/app'
|
||||
export { RerankingModeEnum as HybridSearchModeEnum } from '@/models/datasets'
|
||||
|
||||
export type KnowledgeBaseNodeType = CommonNodeType
|
||||
export enum ChunkStructureEnum {
|
||||
general = 'general',
|
||||
parent_child = 'parent-child',
|
||||
question_answer = 'question-answer',
|
||||
}
|
||||
|
||||
export type RerankingModel = {
|
||||
reranking_provider_name: string
|
||||
reranking_model_name: string
|
||||
}
|
||||
|
||||
export type WeightedScore = {
|
||||
weight_type: WeightedScoreEnum
|
||||
vector_setting: {
|
||||
vector_weight: number
|
||||
embedding_provider_name: string
|
||||
embedding_model_name: string
|
||||
}
|
||||
keyword_setting: {
|
||||
keyword_weight: number
|
||||
}
|
||||
}
|
||||
|
||||
export type RetrievalSetting = {
|
||||
search_method: RETRIEVE_METHOD
|
||||
reranking_enable?: boolean
|
||||
reranking_model?: RerankingModel
|
||||
weights?: WeightedScore
|
||||
top_k: number
|
||||
score_threshold_enabled: boolean
|
||||
score_threshold: number
|
||||
hybridSearchMode: RerankingModeEnum
|
||||
}
|
||||
export type KnowledgeBaseNodeType = CommonNodeType & {
|
||||
index_chunk_variable_selector: string[]
|
||||
chunk_structure: ChunkStructureEnum
|
||||
indexing_technique: IndexingType
|
||||
embedding_model?: string
|
||||
embedding_model_provider?: string
|
||||
keyword_number: number
|
||||
retrieval_model: RetrievalSetting
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue