feat: knowledge base node
parent
12c060b795
commit
a478d95950
@ -1,20 +0,0 @@
|
|||||||
import { FullTextSearch } from '@/app/components/base/icons/src/vender/knowledge'
|
|
||||||
import OptionCard from '../option-card'
|
|
||||||
|
|
||||||
const FullTextSearchCard = () => {
|
|
||||||
return (
|
|
||||||
<OptionCard
|
|
||||||
icon={<FullTextSearch className='h-[15px] w-[15px] text-text-tertiary' />}
|
|
||||||
title='Full-Text Search'
|
|
||||||
description="Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user's query. Users can choose to set weights or configure to a Rerank model."
|
|
||||||
effectColor='purple'
|
|
||||||
>
|
|
||||||
<div className='flex flex-col gap-2'>
|
|
||||||
<div>Vector Search Settings</div>
|
|
||||||
<div>Additional Settings</div>
|
|
||||||
</div>
|
|
||||||
</OptionCard>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export default FullTextSearchCard
|
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
import {
|
||||||
|
FullTextSearch,
|
||||||
|
HybridSearch,
|
||||||
|
VectorSearch,
|
||||||
|
} from '@/app/components/base/icons/src/vender/knowledge'
|
||||||
|
import {
|
||||||
|
HybridSearchModeEnum,
|
||||||
|
RetrievalSearchMethodEnum,
|
||||||
|
} from '../../types'
|
||||||
|
import type {
|
||||||
|
HybridSearchModeOption,
|
||||||
|
Option,
|
||||||
|
} from './type'
|
||||||
|
|
||||||
|
export const useRetrievalSetting = () => {
|
||||||
|
const VectorSearchOption: Option = {
|
||||||
|
id: RetrievalSearchMethodEnum.semantic,
|
||||||
|
icon: VectorSearch as any,
|
||||||
|
title: 'Vector Search',
|
||||||
|
description: 'Generate query embeddings and search for the text chunk most similar to its vector representation.',
|
||||||
|
effectColor: 'purple',
|
||||||
|
}
|
||||||
|
const FullTextSearchOption: Option = {
|
||||||
|
id: RetrievalSearchMethodEnum.fullText,
|
||||||
|
icon: FullTextSearch as any,
|
||||||
|
title: 'Full-Text Search',
|
||||||
|
description: 'Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user\'s query. Users can choose to set weights or configure to a Rerank model.',
|
||||||
|
effectColor: 'purple',
|
||||||
|
}
|
||||||
|
const HybridSearchOption: Option = {
|
||||||
|
id: RetrievalSearchMethodEnum.hybrid,
|
||||||
|
icon: HybridSearch as any,
|
||||||
|
title: 'Hybrid Search',
|
||||||
|
description: 'Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user\'s query. Users can choose to set weights or configure to a Rerank model.',
|
||||||
|
effectColor: 'purple',
|
||||||
|
}
|
||||||
|
|
||||||
|
const options = [
|
||||||
|
VectorSearchOption,
|
||||||
|
FullTextSearchOption,
|
||||||
|
HybridSearchOption,
|
||||||
|
]
|
||||||
|
|
||||||
|
const WeightedScoreModeOption: HybridSearchModeOption = {
|
||||||
|
id: HybridSearchModeEnum.WeightedScore,
|
||||||
|
title: 'Weighted Score',
|
||||||
|
description: 'By adjusting the weights assigned, this rerank strategy determines whether to prioritize semantic or keyword matching.',
|
||||||
|
}
|
||||||
|
|
||||||
|
const RerankModelModeOption: HybridSearchModeOption = {
|
||||||
|
id: HybridSearchModeEnum.RerankingModel,
|
||||||
|
title: 'Rerank Model',
|
||||||
|
description: 'Rerank model will reorder the candidate document list based on the semantic match with user query, improving the results of semantic ranking.',
|
||||||
|
}
|
||||||
|
|
||||||
|
const hybridSearchModeOptions = [
|
||||||
|
WeightedScoreModeOption,
|
||||||
|
RerankModelModeOption,
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
options,
|
||||||
|
hybridSearchModeOptions,
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,21 +0,0 @@
|
|||||||
import { HybridSearch } from '@/app/components/base/icons/src/vender/knowledge'
|
|
||||||
import OptionCard from '../option-card'
|
|
||||||
|
|
||||||
const HybridSearchCard = () => {
|
|
||||||
return (
|
|
||||||
<OptionCard
|
|
||||||
icon={<HybridSearch className='h-[15px] w-[15px] text-text-tertiary' />}
|
|
||||||
title='Hybrid Search'
|
|
||||||
description="Execute full-text search and vector searches simultaneously, re-rank to select the best match for the user's query. Users can choose to set weights or configure to a Rerank model."
|
|
||||||
effectColor='purple'
|
|
||||||
isRecommended
|
|
||||||
>
|
|
||||||
<div className='flex flex-col gap-2'>
|
|
||||||
<div>Vector Search Settings</div>
|
|
||||||
<div>Additional Settings</div>
|
|
||||||
</div>
|
|
||||||
</OptionCard>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export default HybridSearchCard
|
|
||||||
@ -0,0 +1,48 @@
|
|||||||
|
import {
|
||||||
|
memo,
|
||||||
|
useMemo,
|
||||||
|
} from 'react'
|
||||||
|
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
|
||||||
|
import { useModelListAndDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
|
||||||
|
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||||
|
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||||
|
import type { RerankingModel } from '../../types'
|
||||||
|
|
||||||
|
type RerankingModelSelectorProps = {
|
||||||
|
rerankingModel?: RerankingModel
|
||||||
|
onRerankingModelChange?: (model: RerankingModel) => void
|
||||||
|
}
|
||||||
|
const RerankingModelSelector = ({
|
||||||
|
rerankingModel,
|
||||||
|
onRerankingModelChange,
|
||||||
|
}: RerankingModelSelectorProps) => {
|
||||||
|
const {
|
||||||
|
modelList: rerankModelList,
|
||||||
|
} = useModelListAndDefaultModel(ModelTypeEnum.rerank)
|
||||||
|
const rerankModel = useMemo(() => {
|
||||||
|
if (!rerankingModel)
|
||||||
|
return undefined
|
||||||
|
|
||||||
|
return {
|
||||||
|
provider_name: rerankingModel.reranking_provider_name,
|
||||||
|
model_name: rerankingModel.reranking_model_name,
|
||||||
|
}
|
||||||
|
}, [rerankingModel])
|
||||||
|
|
||||||
|
const handleRerankingModelChange = (model: DefaultModel) => {
|
||||||
|
onRerankingModelChange?.({
|
||||||
|
reranking_provider_name: model.provider,
|
||||||
|
reranking_model_name: model.model,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ModelSelector
|
||||||
|
defaultModel={rerankModel && { provider: rerankModel.provider_name, model: rerankModel.model_name }}
|
||||||
|
modelList={rerankModelList}
|
||||||
|
onSelect={handleRerankingModelChange}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default memo(RerankingModelSelector)
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
import { memo } from 'react'
|
||||||
|
import Tooltip from '@/app/components/base/tooltip'
|
||||||
|
import Input from '@/app/components/base/input'
|
||||||
|
import Switch from '@/app/components/base/switch'
|
||||||
|
|
||||||
|
const TopKAndScoreThreshold = () => {
|
||||||
|
return (
|
||||||
|
<div className='grid grid-cols-2 gap-4'>
|
||||||
|
<div>
|
||||||
|
<div className='system-xs-medium mb-0.5 flex h-6 items-center text-text-secondary'>
|
||||||
|
Top k
|
||||||
|
<Tooltip
|
||||||
|
triggerClassName='ml-0.5 shrink-0 w-3.5 h-3.5'
|
||||||
|
popupContent='top k'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<Input
|
||||||
|
type='number'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div className='mb-0.5 flex h-6 items-center'>
|
||||||
|
<Switch
|
||||||
|
className='mr-2'
|
||||||
|
/>
|
||||||
|
<div className='system-sm-medium grow truncate text-text-secondary'>
|
||||||
|
Score Threshold
|
||||||
|
</div>
|
||||||
|
<Tooltip
|
||||||
|
triggerClassName='shrink-0 ml-0.5 w-3.5 h-3.5'
|
||||||
|
popupContent='Score Threshold'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<Input
|
||||||
|
type='number'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default memo(TopKAndScoreThreshold)
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
import type { ComponentType } from 'react'
|
||||||
|
import type {
|
||||||
|
HybridSearchModeEnum,
|
||||||
|
RetrievalSearchMethodEnum,
|
||||||
|
} from '../../types'
|
||||||
|
|
||||||
|
export type Option = {
|
||||||
|
id: RetrievalSearchMethodEnum
|
||||||
|
icon: ComponentType<any>
|
||||||
|
title: any
|
||||||
|
description: string
|
||||||
|
effectColor?: string
|
||||||
|
showEffectColor?: boolean,
|
||||||
|
}
|
||||||
|
|
||||||
|
export type HybridSearchModeOption = {
|
||||||
|
id: HybridSearchModeEnum
|
||||||
|
title: string
|
||||||
|
description: string
|
||||||
|
}
|
||||||
@ -1,20 +0,0 @@
|
|||||||
import { VectorSearch } from '@/app/components/base/icons/src/vender/knowledge'
|
|
||||||
import OptionCard from '../option-card'
|
|
||||||
|
|
||||||
const VectorSearchCard = () => {
|
|
||||||
return (
|
|
||||||
<OptionCard
|
|
||||||
icon={<VectorSearch className='h-[15px] w-[15px] text-text-tertiary' />}
|
|
||||||
title='Vector Search'
|
|
||||||
description='Generate query embeddings and search for the text chunk most similar to its vector representation.'
|
|
||||||
effectColor='purple'
|
|
||||||
>
|
|
||||||
<div className='flex flex-col gap-2'>
|
|
||||||
<div>Vector Search Settings</div>
|
|
||||||
<div>Additional Settings</div>
|
|
||||||
</div>
|
|
||||||
</OptionCard>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export default VectorSearchCard
|
|
||||||
@ -1,19 +1,107 @@
|
|||||||
import {
|
import {
|
||||||
useCallback,
|
useCallback,
|
||||||
useRef,
|
|
||||||
} from 'react'
|
} from 'react'
|
||||||
import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud'
|
import { useStoreApi } from 'reactflow'
|
||||||
import type { KnowledgeBaseNodeType } from '../types'
|
import { useNodeDataUpdate } from '@/app/components/workflow/hooks'
|
||||||
|
import type {
|
||||||
export const useConfig = (id: string, payload: KnowledgeBaseNodeType) => {
|
ChunkStructureEnum,
|
||||||
const {
|
HybridSearchModeEnum,
|
||||||
inputs,
|
IndexMethodEnum,
|
||||||
setInputs,
|
KnowledgeBaseNodeType,
|
||||||
} = useNodeCrud(id, payload)
|
RerankingModel,
|
||||||
const ref = useRef(inputs)
|
RetrievalSearchMethodEnum,
|
||||||
|
} from '../types'
|
||||||
const handleInputsChange = useCallback((newInputs: KnowledgeBaseNodeType) => {
|
|
||||||
setInputs(newInputs)
|
export const useConfig = (id: string) => {
|
||||||
ref.current = newInputs
|
const store = useStoreApi()
|
||||||
}, [setInputs, ref])
|
const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
|
||||||
|
|
||||||
|
const getNodeData = useCallback(() => {
|
||||||
|
const { getNodes } = store.getState()
|
||||||
|
const nodes = getNodes()
|
||||||
|
|
||||||
|
return nodes.find(node => node.id === id)
|
||||||
|
}, [store, id])
|
||||||
|
|
||||||
|
const handleNodeDataUpdate = useCallback((data: Partial<KnowledgeBaseNodeType>) => {
|
||||||
|
handleNodeDataUpdateWithSyncDraft({
|
||||||
|
id,
|
||||||
|
data,
|
||||||
|
})
|
||||||
|
}, [id, handleNodeDataUpdateWithSyncDraft])
|
||||||
|
|
||||||
|
const handleChunkStructureChange = useCallback((chunkStructure: ChunkStructureEnum) => {
|
||||||
|
handleNodeDataUpdate({ chunk_structure: chunkStructure })
|
||||||
|
}, [handleNodeDataUpdate])
|
||||||
|
|
||||||
|
const handleIndexMethodChange = useCallback((indexMethod: IndexMethodEnum) => {
|
||||||
|
handleNodeDataUpdate({ indexing_technique: indexMethod })
|
||||||
|
}, [handleNodeDataUpdate])
|
||||||
|
|
||||||
|
const handleKeywordNumberChange = useCallback((keywordNumber: number) => {
|
||||||
|
handleNodeDataUpdate({ keyword_number: keywordNumber })
|
||||||
|
}, [handleNodeDataUpdate])
|
||||||
|
|
||||||
|
const handleRetrievalSearchMethodChange = useCallback((searchMethod: RetrievalSearchMethodEnum) => {
|
||||||
|
const nodeData = getNodeData()
|
||||||
|
handleNodeDataUpdate({
|
||||||
|
retrieval_model: {
|
||||||
|
...nodeData?.data.retrieval_model,
|
||||||
|
search_method: searchMethod,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}, [getNodeData, handleNodeDataUpdate])
|
||||||
|
|
||||||
|
const handleHybridSearchModeChange = useCallback((hybridSearchMode: HybridSearchModeEnum) => {
|
||||||
|
const nodeData = getNodeData()
|
||||||
|
handleNodeDataUpdate({
|
||||||
|
retrieval_model: {
|
||||||
|
...nodeData?.data.retrieval_model,
|
||||||
|
hybridSearchMode,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}, [getNodeData, handleNodeDataUpdate])
|
||||||
|
|
||||||
|
const handleWeighedScoreChange = useCallback((weightedScore: { value: number[] }) => {
|
||||||
|
const nodeData = getNodeData()
|
||||||
|
handleNodeDataUpdate({
|
||||||
|
retrieval_model: {
|
||||||
|
...nodeData?.data.retrieval_model,
|
||||||
|
weights: {
|
||||||
|
weight_type: 'weighted_score',
|
||||||
|
vector_setting: {
|
||||||
|
vector_weight: weightedScore.value[0],
|
||||||
|
embedding_provider_name: '',
|
||||||
|
embedding_model_name: '',
|
||||||
|
},
|
||||||
|
keyword_setting: {
|
||||||
|
keyword_weight: weightedScore.value[1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}, [getNodeData, handleNodeDataUpdate])
|
||||||
|
|
||||||
|
const handleRerankingModelChange = useCallback((rerankingModel: RerankingModel) => {
|
||||||
|
const nodeData = getNodeData()
|
||||||
|
handleNodeDataUpdate({
|
||||||
|
retrieval_model: {
|
||||||
|
...nodeData?.data.retrieval_model,
|
||||||
|
reranking_model: {
|
||||||
|
reranking_provider_name: rerankingModel.reranking_provider_name,
|
||||||
|
reranking_model_name: rerankingModel.reranking_model_name,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}, [getNodeData, handleNodeDataUpdate])
|
||||||
|
|
||||||
|
return {
|
||||||
|
handleChunkStructureChange,
|
||||||
|
handleIndexMethodChange,
|
||||||
|
handleKeywordNumberChange,
|
||||||
|
handleRetrievalSearchMethodChange,
|
||||||
|
handleHybridSearchModeChange,
|
||||||
|
handleWeighedScoreChange,
|
||||||
|
handleRerankingModelChange,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,3 +1,52 @@
|
|||||||
import type { CommonNodeType } from '@/app/components/workflow/types'
|
import type { CommonNodeType } from '@/app/components/workflow/types'
|
||||||
|
import type { IndexingType } from '@/app/components/datasets/create/step-two'
|
||||||
|
import type { RETRIEVE_METHOD } from '@/types/app'
|
||||||
|
import type { WeightedScoreEnum } from '@/models/datasets'
|
||||||
|
import type { RerankingModeEnum } from '@/models/datasets'
|
||||||
|
export { WeightedScoreEnum } from '@/models/datasets'
|
||||||
|
export { IndexingType as IndexMethodEnum } from '@/app/components/datasets/create/step-two'
|
||||||
|
export { RETRIEVE_METHOD as RetrievalSearchMethodEnum } from '@/types/app'
|
||||||
|
export { RerankingModeEnum as HybridSearchModeEnum } from '@/models/datasets'
|
||||||
|
|
||||||
export type KnowledgeBaseNodeType = CommonNodeType
|
export enum ChunkStructureEnum {
|
||||||
|
general = 'general',
|
||||||
|
parent_child = 'parent-child',
|
||||||
|
question_answer = 'question-answer',
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RerankingModel = {
|
||||||
|
reranking_provider_name: string
|
||||||
|
reranking_model_name: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type WeightedScore = {
|
||||||
|
weight_type: WeightedScoreEnum
|
||||||
|
vector_setting: {
|
||||||
|
vector_weight: number
|
||||||
|
embedding_provider_name: string
|
||||||
|
embedding_model_name: string
|
||||||
|
}
|
||||||
|
keyword_setting: {
|
||||||
|
keyword_weight: number
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RetrievalSetting = {
|
||||||
|
search_method: RETRIEVE_METHOD
|
||||||
|
reranking_enable?: boolean
|
||||||
|
reranking_model?: RerankingModel
|
||||||
|
weights?: WeightedScore
|
||||||
|
top_k: number
|
||||||
|
score_threshold_enabled: boolean
|
||||||
|
score_threshold: number
|
||||||
|
hybridSearchMode: RerankingModeEnum
|
||||||
|
}
|
||||||
|
export type KnowledgeBaseNodeType = CommonNodeType & {
|
||||||
|
index_chunk_variable_selector: string[]
|
||||||
|
chunk_structure: ChunkStructureEnum
|
||||||
|
indexing_technique: IndexingType
|
||||||
|
embedding_model?: string
|
||||||
|
embedding_model_provider?: string
|
||||||
|
keyword_number: number
|
||||||
|
retrieval_model: RetrievalSetting
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue