AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-11 21:09:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: optimize unique document filtering with set (#10082)

Browse Source
This commit is contained in:
omr 2024-10-31 17:32:58 +09:00 committed by GitHub
parent 05d9adeb99
commit 11ca1bec0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
api/core/rag/rerank/rerank_model.py
View File

@ -27,16 +27,15 @@ class RerankModelRunner(BaseRerankRunner):
:return: :return:
""" """
docs = [] docs = []
doc_id = [] doc_id = set()
unique_documents = [] unique_documents = []
dify_documents = [item for item in documents if item.provider == "dify"] for document in documents:
external_documents = [item for item in documents if item.provider == "external"] if document.provider == "dify" and document.metadata["doc_id"] not in doc_id:
for document in dify_documents: doc_id.add(document.metadata["doc_id"])
if document.metadata["doc_id"] not in doc_id:
doc_id.append(document.metadata["doc_id"])
docs.append(document.page_content) docs.append(document.page_content)
unique_documents.append(document) unique_documents.append(document)
for document in external_documents: elif document.provider == "external":
if document not in unique_documents:
docs.append(document.page_content) docs.append(document.page_content)
unique_documents.append(document) unique_documents.append(document)