AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-12 20:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Ignore some emtpy page_content when append to split_documents (#2898)

Browse Source
This commit is contained in:
listeng 2024-03-19 20:55:15 +08:00 committed by GitHub
parent 4419d357c4
commit 696efe494e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/core/rag/index_processor/processor/paragraph_index_processor.py
View File

@ -45,9 +45,10 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
# delete Spliter character # delete Spliter character
page_content = document_node.page_content page_content = document_node.page_content
if page_content.startswith(".") or page_content.startswith("。"): if page_content.startswith(".") or page_content.startswith("。"):
page_content = page_content[1:] page_content = page_content[1:].strip()
else: else:
page_content = page_content page_content = page_content
if len(page_content) > 0:
document_node.page_content = page_content document_node.page_content = page_content
split_documents.append(document_node) split_documents.append(document_node)
all_documents.extend(split_documents) all_documents.extend(split_documents)