AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-15 10:25:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix special code (#473)

Browse Source
This commit is contained in:
Jyong 2023-06-28 13:58:36 +08:00 committed by GitHub
parent 97e9ebd29a
commit 2eea114ac0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api/core/indexing_runner.py
View File

@ -235,7 +235,8 @@ class IndexingRunner:
if len(preview_texts) < 5: if len(preview_texts) < 5:
preview_texts.append(document.page_content) preview_texts.append(document.page_content)
tokens += TokenCalculator.get_num_tokens(self.embedding_model_name, document.page_content) tokens += TokenCalculator.get_num_tokens(self.embedding_model_name,
self.filter_string(document.page_content))
return { return {
"total_segments": total_segments, "total_segments": total_segments,
@ -345,6 +346,8 @@ class IndexingRunner:
return text_docs return text_docs
def filter_string(self, text): def filter_string(self, text):
text = text.replace('<|', '<')
text = text.replace('|>', '>')
pattern = re.compile('[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]') pattern = re.compile('[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]')
return pattern.sub('', text) return pattern.sub('', text)
@ -425,7 +428,7 @@ class IndexingRunner:
return documents return documents
def _split_to_documents(self, text_docs: List[Document], splitter: TextSplitter, def _split_to_documents(self, text_docs: List[Document], splitter: TextSplitter,
processing_rule: DatasetProcessRule) -> List[Document]: processing_rule: DatasetProcessRule) -> List[Document]:
""" """
Split the text documents into nodes. Split the text documents into nodes.
""" """