AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-11 19:58:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: big xls file error (#4859)

Browse Source 
### What problem does this PR solve?

if *.xls file is too large, .eg >50M, I get error.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
SkyfireWXY 2025-02-12 12:39:25 +08:00 committed by GitHub
parent a1cf792245
commit 8fcca1b958
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 72 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
deepdoc/parser/excel_parser.py
View File

@ -1,6 +1,3 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
@ -14,19 +11,51 @@
# limitations under the License. # limitations under the License.
# #
from openpyxl import load_workbook from openpyxl import load_workbook, Workbook
import sys import sys
from io import BytesIO from io import BytesIO
from rag.nlp import find_codec from rag.nlp import find_codec
import pandas as pd
class RAGFlowExcelParser: class RAGFlowExcelParser:
def html(self, fnm, chunk_rows=256): def html(self, fnm, chunk_rows=256):
if isinstance(fnm, str):
wb = load_workbook(fnm) # if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))++
s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else: else:
wb = load_workbook(BytesIO(fnm)) pass
try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
# if len(wb.worksheets) > 0:
# del wb.worksheets[0]
# else: pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass
tb_chunks = [] tb_chunks = []
for sheetname in wb.sheetnames: for sheetname in wb.sheetnames:
@ -60,10 +89,41 @@ class RAGFlowExcelParser:
return tb_chunks return tb_chunks
def __call__(self, fnm): def __call__(self, fnm):
if isinstance(fnm, str): # if isinstance(fnm, str):
wb = load_workbook(fnm) # wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))
s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else: else:
wb = load_workbook(BytesIO(fnm)) pass
try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
if len(wb.worksheets) > 0:
del wb.worksheets[0]
else:
pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass
res = [] res = []
for sheetname in wb.sheetnames: for sheetname in wb.sheetnames:
ws = wb[sheetname] ws = wb[sheetname]
@ -104,3 +164,4 @@ class RAGFlowExcelParser:
if __name__ == "__main__": if __name__ == "__main__":
psr = RAGFlowExcelParser() psr = RAGFlowExcelParser()
psr(sys.argv[1]) psr(sys.argv[1])