AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-14 20:26:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Perf: reduce upload to minio limiter scope (#7878)

Browse Source 
### What problem does this PR solve?
reduce upload_to_minio limter scope

### Type of change
- [x] Performance Improvement
This commit is contained in:
Stephen Hu 2025-05-27 17:49:37 +08:00 committed by GitHub
parent 28cb4df127
commit 273f36cc54
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 20 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
rag/svr/task_executor.py
View File

@ -277,7 +277,6 @@ async def build_chunks(task, progress_callback):
async def upload_to_minio(document, chunk): async def upload_to_minio(document, chunk):
try: try:
async with minio_limiter:
d = copy.deepcopy(document) d = copy.deepcopy(document)
d.update(chunk) d.update(chunk)
d["id"] = xxhash.xxh64((chunk["content_with_weight"] + str(d["doc_id"])).encode("utf-8")).hexdigest() d["id"] = xxhash.xxh64((chunk["content_with_weight"] + str(d["doc_id"])).encode("utf-8")).hexdigest()
@ -294,8 +293,8 @@ async def build_chunks(task, progress_callback):
output_buffer = BytesIO(d["image"]) output_buffer = BytesIO(d["image"])
else: else:
d["image"].save(output_buffer, format='JPEG') d["image"].save(output_buffer, format='JPEG')
async with minio_limiter:
await trio.to_thread.run_sync(lambda: STORAGE_IMPL.put(task["kb_id"], d["id"], output_buffer.getvalue())) await trio.to_thread.run_sync(lambda: STORAGE_IMPL.put(task["kb_id"], d["id"], output_buffer.getvalue()))
d["img_id"] = "{}-{}".format(task["kb_id"], d["id"]) d["img_id"] = "{}-{}".format(task["kb_id"], d["id"])
del d["image"] del d["image"]
docs.append(d) docs.append(d)