AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-13 03:09:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Let task continue dispaching while meeting unexpected doc formats (#199)

Browse Source 
### What problem does this PR solve?

_Briefly describe what this PR aims to solve. Include background context
that will help reviewers understand the purpose of the PR._

Issue link:#[[Link the issue
here](https://github.com/infiniflow/ragflow/issues/198)]

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
- [ ] New Feature (non-breaking change which adds functionality)
- [ ] Breaking Change (fix or feature that could cause existing
functionality not to work as expected)
- [ ] Documentation Update
- [ ] Refactoring
- [ ] Performance Improvement
- [ ] Test cases
- [ ] Python SDK impacted, Need to update PyPI
- [ ] Other (please describe):
This commit is contained in:
KevinHuSh 2024-04-02 11:39:01 +08:00 committed by GitHub
parent 36f2d7b797
commit 572e5b1ff1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 39 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
rag/svr/task_broker.py
View File

@ -73,7 +73,7 @@ def dispatch():
for t in tsks: for t in tsks:
TaskService.delete_by_id(t.id) TaskService.delete_by_id(t.id)
except Exception as e: except Exception as e:
cron_logger.error("delete task exception:" + str(e)) cron_logger.exception(e)
def new_task(): def new_task():
nonlocal r nonlocal r
@ -83,44 +83,48 @@ def dispatch():
} }
tsks = [] tsks = []
if r["type"] == FileType.PDF.value: try:
do_layout = r["parser_config"].get("layout_recognize", True) if r["type"] == FileType.PDF.value:
pages = PdfParser.total_page_number( do_layout = r["parser_config"].get("layout_recognize", True)
r["name"], MINIO.get(r["kb_id"], r["location"])) pages = PdfParser.total_page_number(
page_size = r["parser_config"].get("task_page_size", 12) r["name"], MINIO.get(r["kb_id"], r["location"]))
if r["parser_id"] == "paper": page_size = r["parser_config"].get("task_page_size", 12)
page_size = r["parser_config"].get("task_page_size", 22) if r["parser_id"] == "paper":
if r["parser_id"] == "one": page_size = r["parser_config"].get("task_page_size", 22)
page_size = 1000000000 if r["parser_id"] == "one":
if not do_layout: page_size = 1000000000
page_size = 1000000000 if not do_layout:
page_ranges = r["parser_config"].get("pages") page_size = 1000000000
if not page_ranges: page_ranges = r["parser_config"].get("pages")
page_ranges = [(1, 100000)] if not page_ranges:
for s, e in page_ranges: page_ranges = [(1, 100000)]
s -= 1 for s, e in page_ranges:
s = max(0, s) s -= 1
e = min(e - 1, pages) s = max(0, s)
for p in range(s, e, page_size): e = min(e - 1, pages)
for p in range(s, e, page_size):
task = new_task()
task["from_page"] = p
task["to_page"] = min(p + page_size, e)
tsks.append(task)
elif r["parser_id"] == "table":
rn = HuExcelParser.row_number(
r["name"], MINIO.get(
r["kb_id"], r["location"]))
for i in range(0, rn, 3000):
task = new_task() task = new_task()
task["from_page"] = p task["from_page"] = i
task["to_page"] = min(p + page_size, e) task["to_page"] = min(i + 3000, rn)
tsks.append(task) tsks.append(task)
else:
tsks.append(new_task())
elif r["parser_id"] == "table": bulk_insert_into_db(Task, tsks, True)
rn = HuExcelParser.row_number( set_dispatching(r["id"])
r["name"], MINIO.get( except Exception as e:
r["kb_id"], r["location"])) cron_logger.exception(e)
for i in range(0, rn, 3000):
task = new_task()
task["from_page"] = i
task["to_page"] = min(i + 3000, rn)
tsks.append(task)
else:
tsks.append(new_task())
bulk_insert_into_db(Task, tsks, True)
set_dispatching(r["id"])
tmf.write(str(r["update_time"]) + "\n") tmf.write(str(r["update_time"]) + "\n")
tmf.close() tmf.close()