AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-06-30 09:15:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Improve First Chunk Size (#7806)

Browse Source 
### What problem does this PR solve?

https://github.com/infiniflow/ragflow/issues/7790

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Stephen Hu 2025-05-23 14:30:19 +08:00 committed by GitHub
parent e6cd799d8a
commit db4371c745
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
rag/nlp/__init__.py
View File

@ -524,7 +524,7 @@ def naive_merge(sections, chunk_token_num=128, delimiter="\n。；！？"):
if tnum < 8: if tnum < 8:
pos = "" pos = ""
# Ensure that the length of the merged chunk does not exceed chunk_token_num # Ensure that the length of the merged chunk does not exceed chunk_token_num
if tk_nums[-1] > chunk_token_num: if cks[-1] == "" or tk_nums[-1] > chunk_token_num:
if t.find(pos) < 0: if t.find(pos) < 0:
t += pos t += pos
@ -560,7 +560,7 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。
if tnum < 8: if tnum < 8:
pos = "" pos = ""
# Ensure that the length of the merged chunk does not exceed chunk_token_num # Ensure that the length of the merged chunk does not exceed chunk_token_num
if tk_nums[-1] > chunk_token_num: if cks[-1] == "" or tk_nums[-1] > chunk_token_num:
if t.find(pos) < 0: if t.find(pos) < 0:
t += pos t += pos
cks.append(t) cks.append(t)
@ -627,7 +627,7 @@ def naive_merge_docx(sections, chunk_token_num=128, delimiter="\n。；！？"):
tnum = num_tokens_from_string(t) tnum = num_tokens_from_string(t)
if tnum < 8: if tnum < 8:
pos = "" pos = ""
if tk_nums[-1] > chunk_token_num: if cks[-1] == "" or tk_nums[-1] > chunk_token_num:
if t.find(pos) < 0: if t.find(pos) < 0:
t += pos t += pos
cks.append(t) cks.append(t)