AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-12 06:49:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

add callback to entity extraction (#1843)

Browse Source 
### What problem does this PR solve?

### Type of change

- [x] Refactoring
- [ ]
This commit is contained in:
Kevin Hu 2024-08-07 12:17:02 +08:00 committed by GitHub
parent 06dfb83529
commit a199572bf8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
graphrag/index.py
View File

@ -86,13 +86,15 @@ def build_knowlege_graph_chunks(tenant_id: str, chunks: List[str], callback, ent
for i in range(len(chunks)):
tkn_cnt = num_tokens_from_string(chunks[i])
if cnt+tkn_cnt >= left_token_count and texts:
threads.append(exe.submit(ext, texts, {"entity_types": entity_types}, callback))
for b in range(0, len(texts), 16):
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
texts = []
cnt = 0
texts.append(chunks[i])
cnt += tkn_cnt
if texts:
threads.append(exe.submit(ext, texts))
for b in range(0, len(texts), 16):
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
callback(0.5, "Extracting entities.")
graphs = []