From d6b95e07023924f3b3400729becdc3ce2585eed8 Mon Sep 17 00:00:00 2001 From: "JF.Hsiong" Date: Mon, 14 Apr 2025 09:54:34 +0800 Subject: [PATCH 1/5] feat: We have implemented a complex query mechanism to resolve the issue `17150` by adopting a recursive strategy to support multi-layered nested subqueries. Additionally, to ensure compatibility with historical data, we introduced a new option `metadata_filtering_mode = 'complex_conditions'`, and added a new field `metadata_filtering_complex_conditions` in `KnowledgeRetrievalNodeData` --- .../nodes/knowledge_retrieval/entities.py | 15 +++- .../knowledge_retrieval_node.py | 83 ++++++++++++++++++- .../metadata-filter-selector.tsx | 5 ++ .../nodes/knowledge-retrieval/types.ts | 1 + web/i18n/en-US/workflow.ts | 4 + web/i18n/zh-Hans/workflow.ts | 4 + 6 files changed, 109 insertions(+), 3 deletions(-) diff --git a/api/core/workflow/nodes/knowledge_retrieval/entities.py b/api/core/workflow/nodes/knowledge_retrieval/entities.py index d2e5a15545..53aaadc7f2 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/entities.py +++ b/api/core/workflow/nodes/knowledge_retrieval/entities.py @@ -117,6 +117,18 @@ class MetadataFilteringCondition(BaseModel): conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) +class MetadataFilteringComplexSubCondition(BaseModel): + logical_operator: Optional[Literal["and", "or"]] = "and" + conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) + sub_conditions: Optional[list["MetadataFilteringComplexSubCondition"]] = None + +class MetadataFilteringComplexCondition(BaseModel): + """ + Complex Metadata Filtering Condition. + """ + logical_operator: Optional[Literal["and", "or"]] = "and" + conditions: Optional[list[MetadataFilteringComplexSubCondition]] = Field(default=None, deprecated=True) + class KnowledgeRetrievalNodeData(BaseNodeData): """ Knowledge retrieval Node Data. @@ -128,7 +140,8 @@ class KnowledgeRetrievalNodeData(BaseNodeData): retrieval_mode: Literal["single", "multiple"] multiple_retrieval_config: Optional[MultipleRetrievalConfig] = None single_retrieval_config: Optional[SingleRetrievalConfig] = None - metadata_filtering_mode: Optional[Literal["disabled", "automatic", "manual"]] = "disabled" + metadata_filtering_mode: Optional[Literal["disabled", "automatic", "manual", "complex_conditions"]] = "disabled" metadata_model_config: Optional[ModelConfig] = None metadata_filtering_conditions: Optional[MetadataFilteringCondition] = None + metadata_filtering_complex_conditions: Optional[MetadataFilteringComplexCondition] = None vision: VisionConfig = Field(default_factory=VisionConfig) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 07a711cc4e..ab5e414668 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -44,7 +44,12 @@ from models.dataset import Dataset, DatasetMetadata, Document, RateLimitLog from models.workflow import WorkflowNodeExecutionStatus from services.feature_service import FeatureService -from .entities import KnowledgeRetrievalNodeData, ModelConfig +from .entities import ( + KnowledgeRetrievalNodeData, + MetadataFilteringComplexCondition, + MetadataFilteringComplexSubCondition, + ModelConfig, +) from .exc import ( InvalidModelTypeError, KnowledgeRetrievalNodeError, @@ -315,7 +320,60 @@ class KnowledgeRetrievalNode(LLMNode): for position, item in enumerate(retrieval_resource_list, start=1): item["metadata"]["position"] = position return retrieval_resource_list - + + def _recursive_metadata_filter( + self, metadata_filtering_complex_conditions: MetadataFilteringComplexSubCondition, filters + ): + logical_operator = metadata_filtering_complex_conditions.logical_operator + conditions = metadata_filtering_complex_conditions.conditions + sub_conditions = metadata_filtering_complex_conditions.sub_conditions + + sub_filters = [] + if sub_conditions: + for sub_condition in sub_conditions: + sub_filter = self._recursive_metadata_filter(sub_condition, filters) + sub_filters.append(sub_filter) + + temp_filters = [] + if conditions: + for sequence, condition in enumerate(conditions): + metadata_name = condition.name + expected_value = condition.value + if expected_value is not None or condition.comparison_operator in ("empty", "not empty"): + if isinstance(expected_value, str): + expected_value = self.graph_runtime_state.variable_pool.convert_template( + expected_value + ).value[0] + if expected_value.value_type == "number": # type: ignore + expected_value = expected_value.value # type: ignore + elif expected_value.value_type == "string": # type: ignore + expected_value = re.sub(r"[\r\n\t]+", " ", expected_value.text).strip() # type: ignore + else: + raise ValueError("Invalid expected metadata value type") + temp_filters = self._process_metadata_filter_func( + sequence, + condition.comparison_operator, + metadata_name, + expected_value, + temp_filters, + ) + + if temp_filters: + if logical_operator == "and": # type: ignore + temp_filters = and_(*temp_filters) + else: + temp_filters = or_(*temp_filters) + filters.append(temp_filters) + + if sub_filters: + if logical_operator == "and": # type: ignore + sub_filters = and_(*sub_filters) + else: + sub_filters = or_(*sub_filters) + filters.append(sub_filters) + + return filters + def _get_metadata_filter_condition( self, dataset_ids: list, query: str, node_data: KnowledgeRetrievalNodeData ) -> tuple[Optional[dict[str, list[str]]], Optional[MetadataCondition]]: @@ -329,6 +387,27 @@ class KnowledgeRetrievalNode(LLMNode): metadata_condition = None if node_data.metadata_filtering_mode == "disabled": return None, None + elif node_data.metadata_filtering_mode == "complex_conditions": + # todo: do not support external_knowledge_retrieval + if node_data.metadata_filtering_complex_conditions: + # Enable forward references + MetadataFilteringComplexSubCondition.model_rebuild() + metadata_filtering_complex_conditions = MetadataFilteringComplexCondition( + **node_data.metadata_filtering_complex_conditions.model_dump()) + for sequence, condition in enumerate(metadata_filtering_complex_conditions.conditions): # type: ignore + filters = self._recursive_metadata_filter(condition, filters) + if filters: + if metadata_filtering_complex_conditions.logical_operator == "and": # type: ignore + document_query = document_query.filter(and_(*filters)) + else: + document_query = document_query.filter(or_(*filters)) + metadata_condition = metadata_filtering_complex_conditions + documents = document_query.all() + # group by dataset_id + metadata_filter_document_ids = defaultdict(list) if documents else None # type: ignore + for document in documents: + metadata_filter_document_ids[document.dataset_id].append(document.id) # type: ignore + return metadata_filter_document_ids, metadata_condition elif node_data.metadata_filtering_mode == "automatic": automatic_metadata_filters = self._automatic_metadata_filter_func(dataset_ids, query, node_data) if automatic_metadata_filters: diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-filter/metadata-filter-selector.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-filter/metadata-filter-selector.tsx index 7183e685f4..4bc0f868e8 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-filter/metadata-filter-selector.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-filter/metadata-filter-selector.tsx @@ -38,6 +38,11 @@ const MetadataFilterSelector = ({ value: t('workflow.nodes.knowledgeRetrieval.metadata.options.manual.title'), desc: t('workflow.nodes.knowledgeRetrieval.metadata.options.manual.subTitle'), }, + { + key: MetadataFilteringModeEnum.complexConditions, + value: t('workflow.nodes.knowledgeRetrieval.metadata.options.complexConditions.title'), + desc: t('workflow.nodes.knowledgeRetrieval.metadata.options.complexConditions.subTitle'), + }, ] const selectedOption = options.find(option => option.key === value)! diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts index 1cae4ecd3b..b19ddaddfb 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts @@ -74,6 +74,7 @@ export enum MetadataFilteringModeEnum { disabled = 'disabled', automatic = 'automatic', manual = 'manual', + complexConditions = 'complex_conditions', } export enum MetadataFilteringVariableType { diff --git a/web/i18n/en-US/workflow.ts b/web/i18n/en-US/workflow.ts index 939229caeb..3041e94bcc 100644 --- a/web/i18n/en-US/workflow.ts +++ b/web/i18n/en-US/workflow.ts @@ -448,6 +448,10 @@ const translation = { title: 'Manual', subTitle: 'Manually add metadata filtering conditions', }, + complexConditions: { + title: 'complexConditions', + subTitle: 'Manually add metadata filtering complex conditions', + }, }, panel: { title: 'Metadata Filter Conditions', diff --git a/web/i18n/zh-Hans/workflow.ts b/web/i18n/zh-Hans/workflow.ts index 26d847f306..03abd7ae32 100644 --- a/web/i18n/zh-Hans/workflow.ts +++ b/web/i18n/zh-Hans/workflow.ts @@ -449,6 +449,10 @@ const translation = { title: '手动', subTitle: '手动添加元数据过滤条件', }, + complexConditions: { + title: '手动多重条件', + subTitle: '手动添加元数据多重过滤条件', + }, }, panel: { title: '元数据过滤条件', From 96950fd8b50cfd61d2388a7ea5e9f2932de99666 Mon Sep 17 00:00:00 2001 From: "JF.Hsiong" Date: Mon, 14 Apr 2025 12:14:25 +0800 Subject: [PATCH 2/5] style: fix style issues --- .../nodes/knowledge_retrieval/entities.py | 3 ++ .../knowledge_retrieval_node.py | 49 ++++++++++--------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/api/core/workflow/nodes/knowledge_retrieval/entities.py b/api/core/workflow/nodes/knowledge_retrieval/entities.py index 53aaadc7f2..6f45af1a14 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/entities.py +++ b/api/core/workflow/nodes/knowledge_retrieval/entities.py @@ -122,13 +122,16 @@ class MetadataFilteringComplexSubCondition(BaseModel): conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) sub_conditions: Optional[list["MetadataFilteringComplexSubCondition"]] = None + class MetadataFilteringComplexCondition(BaseModel): """ Complex Metadata Filtering Condition. """ + logical_operator: Optional[Literal["and", "or"]] = "and" conditions: Optional[list[MetadataFilteringComplexSubCondition]] = Field(default=None, deprecated=True) + class KnowledgeRetrievalNodeData(BaseNodeData): """ Knowledge retrieval Node Data. diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index ab5e414668..4e5cc56ea6 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -6,7 +6,7 @@ from collections import defaultdict from collections.abc import Mapping, Sequence from typing import Any, Optional, cast -from sqlalchemy import Integer, and_, func, or_, text +from sqlalchemy import ColumnElement, Integer, and_, func, or_, text from sqlalchemy import cast as sqlalchemy_cast from core.app.app_config.entities import DatasetRetrieveConfigEntity @@ -320,30 +320,30 @@ class KnowledgeRetrievalNode(LLMNode): for position, item in enumerate(retrieval_resource_list, start=1): item["metadata"]["position"] = position return retrieval_resource_list - + def _recursive_metadata_filter( - self, metadata_filtering_complex_conditions: MetadataFilteringComplexSubCondition, filters + self, metadata_filtering_complex_conditions: MetadataFilteringComplexSubCondition, filters ): logical_operator = metadata_filtering_complex_conditions.logical_operator conditions = metadata_filtering_complex_conditions.conditions sub_conditions = metadata_filtering_complex_conditions.sub_conditions - + sub_filters = [] if sub_conditions: for sub_condition in sub_conditions: sub_filter = self._recursive_metadata_filter(sub_condition, filters) sub_filters.append(sub_filter) - - temp_filters = [] + + temp_filters: list = [] if conditions: for sequence, condition in enumerate(conditions): metadata_name = condition.name expected_value = condition.value if expected_value is not None or condition.comparison_operator in ("empty", "not empty"): if isinstance(expected_value, str): - expected_value = self.graph_runtime_state.variable_pool.convert_template( - expected_value - ).value[0] + expected_value = self.graph_runtime_state.variable_pool.convert_template(expected_value).value[ + 0 + ] if expected_value.value_type == "number": # type: ignore expected_value = expected_value.value # type: ignore elif expected_value.value_type == "string": # type: ignore @@ -357,23 +357,24 @@ class KnowledgeRetrievalNode(LLMNode): expected_value, temp_filters, ) - + temp_filters_result: ColumnElement[bool] if temp_filters: if logical_operator == "and": # type: ignore - temp_filters = and_(*temp_filters) + temp_filters_result = and_(*temp_filters) else: - temp_filters = or_(*temp_filters) - filters.append(temp_filters) - + temp_filters_result = or_(*temp_filters) + filters.append(temp_filters_result) + + sub_filters_result: ColumnElement[bool] if sub_filters: if logical_operator == "and": # type: ignore - sub_filters = and_(*sub_filters) + sub_filters_result = and_(*sub_filters) else: - sub_filters = or_(*sub_filters) - filters.append(sub_filters) - + sub_filters_result = or_(*sub_filters) + filters.append(sub_filters_result) + return filters - + def _get_metadata_filter_condition( self, dataset_ids: list, query: str, node_data: KnowledgeRetrievalNodeData ) -> tuple[Optional[dict[str, list[str]]], Optional[MetadataCondition]]: @@ -393,21 +394,21 @@ class KnowledgeRetrievalNode(LLMNode): # Enable forward references MetadataFilteringComplexSubCondition.model_rebuild() metadata_filtering_complex_conditions = MetadataFilteringComplexCondition( - **node_data.metadata_filtering_complex_conditions.model_dump()) - for sequence, condition in enumerate(metadata_filtering_complex_conditions.conditions): # type: ignore + **node_data.metadata_filtering_complex_conditions.model_dump() + ) + for condition in metadata_filtering_complex_conditions.conditions: # type: ignore filters = self._recursive_metadata_filter(condition, filters) if filters: if metadata_filtering_complex_conditions.logical_operator == "and": # type: ignore document_query = document_query.filter(and_(*filters)) else: document_query = document_query.filter(or_(*filters)) - metadata_condition = metadata_filtering_complex_conditions documents = document_query.all() # group by dataset_id metadata_filter_document_ids = defaultdict(list) if documents else None # type: ignore for document in documents: metadata_filter_document_ids[document.dataset_id].append(document.id) # type: ignore - return metadata_filter_document_ids, metadata_condition + return metadata_filter_document_ids, MetadataCondition() elif node_data.metadata_filtering_mode == "automatic": automatic_metadata_filters = self._automatic_metadata_filter_func(dataset_ids, query, node_data) if automatic_metadata_filters: @@ -435,7 +436,7 @@ class KnowledgeRetrievalNode(LLMNode): if node_data.metadata_filtering_conditions: metadata_condition = MetadataCondition(**node_data.metadata_filtering_conditions.model_dump()) if node_data.metadata_filtering_conditions: - for sequence, condition in enumerate(node_data.metadata_filtering_conditions.conditions): # type: ignore + for sequence, condition in enumerate(metadata_condition.conditions): # type: ignore metadata_name = condition.name expected_value = condition.value if expected_value is not None or condition.comparison_operator in ("empty", "not empty"): From 96bbc1b68278d80397044705593dc9145134fd47 Mon Sep 17 00:00:00 2001 From: "JF.Hsiong" Date: Fri, 18 Apr 2025 15:51:02 +0800 Subject: [PATCH 3/5] feat: optimize metadata recursive strategy --- .../nodes/knowledge_retrieval/entities.py | 14 +------ .../knowledge_retrieval_node.py | 37 +++++++++++-------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/api/core/workflow/nodes/knowledge_retrieval/entities.py b/api/core/workflow/nodes/knowledge_retrieval/entities.py index 6f45af1a14..7fde733eb5 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/entities.py +++ b/api/core/workflow/nodes/knowledge_retrieval/entities.py @@ -116,20 +116,10 @@ class MetadataFilteringCondition(BaseModel): logical_operator: Optional[Literal["and", "or"]] = "and" conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) - -class MetadataFilteringComplexSubCondition(BaseModel): - logical_operator: Optional[Literal["and", "or"]] = "and" - conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) - sub_conditions: Optional[list["MetadataFilteringComplexSubCondition"]] = None - - class MetadataFilteringComplexCondition(BaseModel): - """ - Complex Metadata Filtering Condition. - """ - logical_operator: Optional[Literal["and", "or"]] = "and" - conditions: Optional[list[MetadataFilteringComplexSubCondition]] = Field(default=None, deprecated=True) + conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) + sub_conditions: Optional[list["MetadataFilteringComplexCondition"]] = None class KnowledgeRetrievalNodeData(BaseNodeData): diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 4e5cc56ea6..2af78c72e0 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -47,7 +47,6 @@ from services.feature_service import FeatureService from .entities import ( KnowledgeRetrievalNodeData, MetadataFilteringComplexCondition, - MetadataFilteringComplexSubCondition, ModelConfig, ) from .exc import ( @@ -322,7 +321,7 @@ class KnowledgeRetrievalNode(LLMNode): return retrieval_resource_list def _recursive_metadata_filter( - self, metadata_filtering_complex_conditions: MetadataFilteringComplexSubCondition, filters + self, metadata_filtering_complex_conditions: MetadataFilteringComplexCondition, filters ): logical_operator = metadata_filtering_complex_conditions.logical_operator conditions = metadata_filtering_complex_conditions.conditions @@ -331,8 +330,8 @@ class KnowledgeRetrievalNode(LLMNode): sub_filters = [] if sub_conditions: for sub_condition in sub_conditions: - sub_filter = self._recursive_metadata_filter(sub_condition, filters) - sub_filters.append(sub_filter) + sub_filter = self._recursive_metadata_filter(sub_condition, []) + sub_filters.extend(sub_filter) temp_filters: list = [] if conditions: @@ -357,16 +356,27 @@ class KnowledgeRetrievalNode(LLMNode): expected_value, temp_filters, ) - temp_filters_result: ColumnElement[bool] - if temp_filters: + + sub_filters_result: ColumnElement + temp_filters_result: ColumnElement + if temp_filters and sub_filters: + temp_all_filters = sub_filters +temp_filters + if logical_operator == "and": # type: ignore + sub_filters_result = and_(*temp_all_filters) + else: + sub_filters_result = or_(*temp_all_filters) + filters.append(sub_filters_result) + return filters + + if temp_filters: # text if logical_operator == "and": # type: ignore temp_filters_result = and_(*temp_filters) else: temp_filters_result = or_(*temp_filters) filters.append(temp_filters_result) + return filters - sub_filters_result: ColumnElement[bool] - if sub_filters: + if sub_filters: # Boolean if logical_operator == "and": # type: ignore sub_filters_result = and_(*sub_filters) else: @@ -375,6 +385,7 @@ class KnowledgeRetrievalNode(LLMNode): return filters + def _get_metadata_filter_condition( self, dataset_ids: list, query: str, node_data: KnowledgeRetrievalNodeData ) -> tuple[Optional[dict[str, list[str]]], Optional[MetadataCondition]]: @@ -392,17 +403,13 @@ class KnowledgeRetrievalNode(LLMNode): # todo: do not support external_knowledge_retrieval if node_data.metadata_filtering_complex_conditions: # Enable forward references - MetadataFilteringComplexSubCondition.model_rebuild() + MetadataFilteringComplexCondition.model_rebuild() metadata_filtering_complex_conditions = MetadataFilteringComplexCondition( **node_data.metadata_filtering_complex_conditions.model_dump() ) - for condition in metadata_filtering_complex_conditions.conditions: # type: ignore - filters = self._recursive_metadata_filter(condition, filters) + filters = self._recursive_metadata_filter(metadata_filtering_complex_conditions, filters) if filters: - if metadata_filtering_complex_conditions.logical_operator == "and": # type: ignore - document_query = document_query.filter(and_(*filters)) - else: - document_query = document_query.filter(or_(*filters)) + document_query = document_query.filter(*filters) documents = document_query.all() # group by dataset_id metadata_filter_document_ids = defaultdict(list) if documents else None # type: ignore From d2b2489bbf6ffbe81658c2c5a4c6b2f25ce307c2 Mon Sep 17 00:00:00 2001 From: "JF.Hsiong" Date: Fri, 18 Apr 2025 16:06:38 +0800 Subject: [PATCH 4/5] style: fix style issues --- .../workflow/nodes/knowledge_retrieval/entities.py | 1 + .../knowledge_retrieval/knowledge_retrieval_node.py | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/core/workflow/nodes/knowledge_retrieval/entities.py b/api/core/workflow/nodes/knowledge_retrieval/entities.py index 7fde733eb5..07dca08129 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/entities.py +++ b/api/core/workflow/nodes/knowledge_retrieval/entities.py @@ -116,6 +116,7 @@ class MetadataFilteringCondition(BaseModel): logical_operator: Optional[Literal["and", "or"]] = "and" conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) + class MetadataFilteringComplexCondition(BaseModel): logical_operator: Optional[Literal["and", "or"]] = "and" conditions: Optional[list[Condition]] = Field(default=None, deprecated=True) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 2af78c72e0..bda728812c 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -356,19 +356,19 @@ class KnowledgeRetrievalNode(LLMNode): expected_value, temp_filters, ) - + sub_filters_result: ColumnElement temp_filters_result: ColumnElement if temp_filters and sub_filters: - temp_all_filters = sub_filters +temp_filters + temp_all_filters = sub_filters + temp_filters if logical_operator == "and": # type: ignore sub_filters_result = and_(*temp_all_filters) else: sub_filters_result = or_(*temp_all_filters) filters.append(sub_filters_result) return filters - - if temp_filters: # text + + if temp_filters: # text if logical_operator == "and": # type: ignore temp_filters_result = and_(*temp_filters) else: @@ -376,7 +376,7 @@ class KnowledgeRetrievalNode(LLMNode): filters.append(temp_filters_result) return filters - if sub_filters: # Boolean + if sub_filters: # Boolean if logical_operator == "and": # type: ignore sub_filters_result = and_(*sub_filters) else: @@ -385,7 +385,6 @@ class KnowledgeRetrievalNode(LLMNode): return filters - def _get_metadata_filter_condition( self, dataset_ids: list, query: str, node_data: KnowledgeRetrievalNodeData ) -> tuple[Optional[dict[str, list[str]]], Optional[MetadataCondition]]: From c326b2f2d654c51823c3ad82a71a1898ad3c5d12 Mon Sep 17 00:00:00 2001 From: "JF.Hsiong" Date: Fri, 18 Apr 2025 19:16:45 +0800 Subject: [PATCH 5/5] feat: optimize metadata recursive strategy --- .../knowledge_retrieval/knowledge_retrieval_node.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index bda728812c..9f95b0466b 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -360,11 +360,14 @@ class KnowledgeRetrievalNode(LLMNode): sub_filters_result: ColumnElement temp_filters_result: ColumnElement if temp_filters and sub_filters: - temp_all_filters = sub_filters + temp_filters if logical_operator == "and": # type: ignore - sub_filters_result = and_(*temp_all_filters) + all_sub_filters = and_(*sub_filters) + all_temp_filters = and_(*temp_filters) + sub_filters_result = and_(all_temp_filters, all_sub_filters) else: - sub_filters_result = or_(*temp_all_filters) + all_sub_filters = or_(*sub_filters) + all_temp_filters = or_(*temp_filters) + sub_filters_result = or_(all_sub_filters, all_temp_filters) filters.append(sub_filters_result) return filters