AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-18 00:55:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix/filter empty segment (#1004)

Browse Source 
Co-authored-by: jyong <jyong@dify.ai>
This commit is contained in:
Jyong 2023-08-25 15:50:29 +08:00 committed by GitHub
parent e7199826cc
commit 2d604d9330
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
api/core/indexing_runner.py
View File

@ -525,12 +525,13 @@ class IndexingRunner:
documents = splitter.split_documents([text_doc]) documents = splitter.split_documents([text_doc])
split_documents = [] split_documents = []
for document_node in documents: for document_node in documents:
doc_id = str(uuid.uuid4())
hash = helper.generate_text_hash(document_node.page_content)
document_node.metadata['doc_id'] = doc_id
document_node.metadata['doc_hash'] = hash
split_documents.append(document_node) if document_node.page_content.strip():
doc_id = str(uuid.uuid4())
hash = helper.generate_text_hash(document_node.page_content)
document_node.metadata['doc_id'] = doc_id
document_node.metadata['doc_hash'] = hash
split_documents.append(document_node)
all_documents.extend(split_documents) all_documents.extend(split_documents)
# processing qa document # processing qa document
if document_form == 'qa_model': if document_form == 'qa_model':

4
api/services/dataset_service.py
View File

@ -891,6 +891,10 @@ class SegmentService:
if document.doc_form == 'qa_model': if document.doc_form == 'qa_model':
if 'answer' not in args or not args['answer']: if 'answer' not in args or not args['answer']:
raise ValueError("Answer is required") raise ValueError("Answer is required")
if not args['answer'].strip():
raise ValueError("Answer is empty")
if 'content' not in args or not args['content'] or not args['content'].strip():
raise ValueError("Content is empty")
@classmethod @classmethod
def create_segment(cls, args: dict, document: Document, dataset: Dataset): def create_segment(cls, args: dict, document: Document, dataset: Dataset):