AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-13 03:09:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

add callback to entity extraction (#1843)

Browse Source 
### What problem does this PR solve?

### Type of change

- [x] Refactoring
- [ ]
This commit is contained in:
Kevin Hu 2024-08-07 12:17:02 +08:00 committed by GitHub
parent 06dfb83529
commit a199572bf8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
graphrag/index.py
View File

@ -86,13 +86,15 @@ def build_knowlege_graph_chunks(tenant_id: str, chunks: List[str], callback, ent
for i in range(len(chunks)): for i in range(len(chunks)):
tkn_cnt = num_tokens_from_string(chunks[i]) tkn_cnt = num_tokens_from_string(chunks[i])
if cnt+tkn_cnt >= left_token_count and texts: if cnt+tkn_cnt >= left_token_count and texts:
threads.append(exe.submit(ext, texts, {"entity_types": entity_types}, callback)) for b in range(0, len(texts), 16):
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
texts = [] texts = []
cnt = 0 cnt = 0
texts.append(chunks[i]) texts.append(chunks[i])
cnt += tkn_cnt cnt += tkn_cnt
if texts: if texts:
threads.append(exe.submit(ext, texts)) for b in range(0, len(texts), 16):
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
callback(0.5, "Extracting entities.") callback(0.5, "Extracting entities.")
graphs = [] graphs = []