fix: optimize unique document filtering with set (#10082)

api/core/rag/rerank/rerank_model.py

@@ -27,16 +27,15 @@ class RerankModelRunner(BaseRerankRunner):
         :return:
         """
         docs = []
-        doc_id = []
+        doc_id = set()
         unique_documents = []
-        dify_documents = [item for item in documents if item.provider == "dify"]
+        for document in documents:
-        external_documents = [item for item in documents if item.provider == "external"]
+            if document.provider == "dify" and document.metadata["doc_id"] not in doc_id:
-        for document in dify_documents:
+                doc_id.add(document.metadata["doc_id"])
-            if document.metadata["doc_id"] not in doc_id:
-                doc_id.append(document.metadata["doc_id"])
                 docs.append(document.page_content)
                 unique_documents.append(document)
-        for document in external_documents:
+            elif document.provider == "external":
+                if document not in unique_documents:
                     docs.append(document.page_content)
                     unique_documents.append(document)