AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-15 00:16:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

chunk-overlap None check (#2781)

Browse Source 
Co-authored-by: jyong <jyong@dify.ai>
This commit is contained in:
Jyong 2024-03-11 15:36:56 +08:00 committed by GitHub
parent e184c8cb42
commit 6454e1d644
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api/core/indexing_runner.py
View File

@ -416,9 +416,14 @@ class IndexingRunner:
if separator: if separator:
separator = separator.replace('\\n', '\n') separator = separator.replace('\\n', '\n')
if 'chunk_overlap' in segmentation and segmentation['chunk_overlap']:
chunk_overlap = segmentation['chunk_overlap']
else:
chunk_overlap = 0
character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder( character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder(
chunk_size=segmentation["max_tokens"], chunk_size=segmentation["max_tokens"],
chunk_overlap=segmentation.get('chunk_overlap', 0), chunk_overlap=chunk_overlap,
fixed_separator=separator, fixed_separator=separator,
separators=["\n\n", "。", ".", " ", ""], separators=["\n\n", "。", ".", " ", ""],
embedding_model_instance=embedding_model_instance embedding_model_instance=embedding_model_instance