feat(kb api): retrieve api add metadata_filtering_conditions parameters

api/core/rag/retrieval/dataset_retrieval.py

@@ -936,6 +936,9 @@ class DatasetRetrieval:
         return metadata_filter_document_ids, metadata_condition
 
     def _replace_metadata_filter_value(self, text: str, inputs: dict) -> str:
+        if not inputs:
+            return text
+
         def replacer(match):
             key = match.group(1)
             return str(inputs.get(key, f"{{{{{key}}}}}"))

api/services/hit_testing_service.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from core.rag.datasource.retrieval_service import RetrievalService
 from core.rag.models.document import Document
+from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
 from core.rag.retrieval.retrieval_methods import RetrievalMethod
 from extensions.ext_database import db
 from models.account import Account
@@ -34,7 +35,28 @@ class HitTestingService:
         # get retrieval model , if the model is not setting , using default
         if not retrieval_model:
             retrieval_model = dataset.retrieval_model or default_retrieval_model
+        document_ids_filter = None
+        metadata_filtering_conditions = retrieval_model.get("metadata_filtering_conditions", {})
+        if metadata_filtering_conditions:
+            dataset_retrieval = DatasetRetrieval()
 
+            from core.app.app_config.entities import MetadataFilteringCondition
+            metadata_filtering_conditions = MetadataFilteringCondition(**metadata_filtering_conditions)
+
+            metadata_filter_document_ids, metadata_condition = dataset_retrieval.get_metadata_filter_condition(
+                dataset_ids=[dataset.id],
+                query=query,
+                metadata_filtering_mode="manual",
+                metadata_filtering_conditions=metadata_filtering_conditions,
+                inputs={},
+                tenant_id=None,
+                user_id=None,
+                metadata_model_config=None
+            )
+            if metadata_filter_document_ids:
+                document_ids_filter = metadata_filter_document_ids.get(dataset.id, [])
+            if metadata_condition and not document_ids_filter:
+                return cls.compact_retrieve_response(query, [])
         all_documents = RetrievalService.retrieve(
             retrieval_method=retrieval_model.get("search_method", "semantic_search"),
             dataset_id=dataset.id,
@@ -48,6 +70,7 @@ class HitTestingService:
             else None,
             reranking_mode=retrieval_model.get("reranking_mode") or "reranking_model",
             weights=retrieval_model.get("weights", None),
+            document_ids_filter=document_ids_filter
         )
 
         end = time.perf_counter()

web/app/(commonLayout)/datasets/template/template.en.mdx

@@ -1841,7 +1841,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
         Query keyword
       </Property>
       <Property name='retrieval_model' type='object' key='retrieval_model'>
-        Retrieval model (optional, if not filled, it will be recalled according to the default method)
+        Retrieval parameters (optional, if not filled, it will be recalled according to the default method)
           - <code>search_method</code> (text) Search method: One of the following four keywords is required
             - <code>keyword_search</code> Keyword search
             - <code>semantic_search</code> Semantic search
@@ -1855,6 +1855,31 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
           - <code>top_k</code> (integer) Number of results to return (optional)
           - <code>score_threshold_enabled</code> (bool) Whether to enable score threshold
           - <code>score_threshold</code> (float) Score threshold
+          - <code>metadata_filtering_conditions</code> (object) Metadata filtering conditions
+            - <code>logical_operator</code> (string) Logical operator: <code>and</code> | <code>or</code>
+            - <code>conditions</code> (array[object]) Conditions list
+              - <code>name</code> (string) Metadata field name
+              - <code>comparison_operator</code> (string) Comparison operator, allowed values:
+                - String comparison:
+                  - <code>contains</code>: Contains
+                  - <code>not contains</code>: Does not contain
+                  - <code>start with</code>: Starts with
+                  - <code>end with</code>: Ends with
+                  - <code>is</code>: Equals
+                  - <code>is not</code>: Does not equal
+                  - <code>empty</code>: Is empty
+                  - <code>not empty</code>: Is not empty
+                - Numeric comparison:
+                  - <code>=</code>: Equals
+                  - <code>≠</code>: Does not equal
+                  - <code>></code>: Greater than
+                  - <code>< </code>: Less than
+                  - <code>≥</code>: Greater than or equal
+                  - <code>≤</code>: Less than or equal
+                - Time comparison:
+                  - <code>before</code>: Before
+                  - <code>after</code>: After
+              - <code>value</code> (string|number|null) Comparison value
       </Property>
       <Property name='external_retrieval_model' type='object' key='external_retrieval_model'>
           Unused field
@@ -1879,7 +1904,17 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
         "weights": null,
         "top_k": 1,
         "score_threshold_enabled": false,
-        "score_threshold": null
+        "score_threshold": null,
+        "metadata_filtering_conditions": {
+            "logical_operator": "and",
+            "conditions": [
+                {
+                    "name": "document_name",
+                    "comparison_operator": "contains",
+                    "value": "test"
+                }
+            ]
+        }
     }
 }'`}
     >
@@ -2159,9 +2194,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
       label="/datasets/{dataset_id}/documents/metadata"
       targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/metadata' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json'\\\n--data-raw '{"operation_data": [{"document_id": "document_id", "metadata_list": [{"id": "id", "value": "value", "name": "name"}]}]}'`}
     >
-    ```bash {{ title: 'cURL' }}
+    ```bash {{ title: 'cURL' }}    </CodeGroup>
-    ```
-    </CodeGroup>
   </Col>
 </Row>
 

web/app/(commonLayout)/datasets/template/template.ja.mdx

@@ -1596,20 +1596,45 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
         クエリキーワード
       </Property>
       <Property name='retrieval_model' type='object' key='retrieval_model'>
-        検索モデル (オプション、入力されない場合はデフォルトの方法でリコールされます)
+        検索パラメータ（オプション、入力されない場合はデフォルトの方法でリコールされます）
           - <code>search_method</code> (text) 検索方法: 以下の4つのキーワードのいずれかが必要です
             - <code>keyword_search</code> キーワード検索
             - <code>semantic_search</code> セマンティック検索
             - <code>full_text_search</code> 全文検索
             - <code>hybrid_search</code> ハイブリッド検索
-        - <code>reranking_enable</code> (bool) 再ランキングを有効にするかどうか、検索モードが semantic_search または hybrid_search の場合に必須 (オプション)
+          - <code>reranking_enable</code> (bool) 再ランキングを有効にするかどうか、検索モードがsemantic_searchまたはhybrid_searchの場合に必須（オプション）
           - <code>reranking_mode</code> (object) 再ランキングモデル構成、再ランキングが有効な場合に必須
               - <code>reranking_provider_name</code> (string) 再ランキングモデルプロバイダー
               - <code>reranking_model_name</code> (string) 再ランキングモデル名
           - <code>weights</code> (float) ハイブリッド検索モードでのセマンティック検索の重み設定
-        - <code>top_k</code> (integer) 返される結果の数 (オプション)
+          - <code>top_k</code> (integer) 返される結果の数（オプション）
           - <code>score_threshold_enabled</code> (bool) スコア閾値を有効にするかどうか
           - <code>score_threshold</code> (float) スコア閾値
+          - <code>metadata_filtering_conditions</code> (object) メタデータフィルタリング条件
+            - <code>logical_operator</code> (string) 論理演算子: <code>and</code> | <code>or</code>
+            - <code>conditions</code> (array[object]) 条件リスト
+              - <code>name</code> (string) メタデータフィールド名
+              - <code>comparison_operator</code> (string) 比較演算子、許可される値:
+                - 文字列比較:
+                  - <code>contains</code>: 含む
+                  - <code>not contains</code>: 含まない
+                  - <code>start with</code>: で始まる
+                  - <code>end with</code>: で終わる
+                  - <code>is</code>: 等しい
+                  - <code>is not</code>: 等しくない
+                  - <code>empty</code>: 空
+                  - <code>not empty</code>: 空でない
+                - 数値比較:
+                  - <code>=</code>: 等しい
+                  - <code>≠</code>: 等しくない
+                  - <code>></code>: より大きい
+                  - <code>< </code>: より小さい
+                  - <code>≥</code>: 以上
+                  - <code>≤</code>: 以下
+                - 時間比較:
+                  - <code>before</code>: より前
+                  - <code>after</code>: より後
+              - <code>value</code> (string|number|null) 比較値
       </Property>
       <Property name='external_retrieval_model' type='object' key='external_retrieval_model'>
           未使用フィールド
@@ -1634,7 +1659,17 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
         "weights": null,
         "top_k": 1,
         "score_threshold_enabled": false,
-        "score_threshold": null
+        "score_threshold": null,
+        "metadata_filtering_conditions": {
+            "logical_operator": "and",
+            "conditions": [
+                {
+                    "name": "document_name",
+                    "comparison_operator": "contains",
+                    "value": "test"
+                }
+            ]
+        }
     }
 }'`}
     >

web/app/(commonLayout)/datasets/template/template.zh.mdx

@@ -1896,6 +1896,31 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
         - <code>top_k</code> (integer) 返回结果数量，非必填
         - <code>score_threshold_enabled</code> (bool) 是否开启 score 阈值
         - <code>score_threshold</code> (float) Score 阈值
+        - <code>metadata_filtering_conditions</code> (object) 元数据过滤条件
+          - <code>logical_operator</code> (string) 逻辑运算符: <code>and</code> | <code>or</code>
+          - <code>conditions</code> (array[object]) 条件列表
+            - <code>name</code> (string) 元数据字段名
+            - <code>comparison_operator</code> (string) 比较运算符，可选值:
+              - 字符串比较:
+                - <code>contains</code>: 包含
+                - <code>not contains</code>: 不包含
+                - <code>start with</code>: 以...开头
+                - <code>end with</code>: 以...结尾
+                - <code>is</code>: 等于
+                - <code>is not</code>: 不等于
+                - <code>empty</code>: 为空
+                - <code>not empty</code>: 不为空
+              - 数值比较:
+                - <code>=</code>: 等于
+                - <code>≠</code>: 不等于
+                - <code>></code>: 大于
+                - <code> < </code>: 小于
+                - <code>≥</code>: 大于等于
+                - <code>≤</code>: 小于等于
+              - 时间比较:
+                - <code>before</code>: 早于
+                - <code>after</code>: 晚于
+            - <code>value</code> (string|number|null) 比较值
       </Property>
       <Property name='external_retrieval_model' type='object' key='external_retrieval_model'>
           未启用字段
@@ -1920,7 +1945,17 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
         "weights": null,
         "top_k": 1,
         "score_threshold_enabled": false,
-        "score_threshold": null
+        "score_threshold": null,
+        "metadata_filtering_conditions": {
+            "logical_operator": "and",
+            "conditions": [
+                {
+                    "name": "document_name",
+                    "comparison_operator": "contains",
+                    "value": "test"
+                }
+            ]
+        }
     }
 }'`}
     >