mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-08-12 02:19:05 +08:00
fix: big xls file error (#4859)
### What problem does this PR solve? if *.xls file is too large, .eg >50M, I get error. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
parent
a1cf792245
commit
8fcca1b958
@ -1,6 +1,3 @@
|
|||||||
#
|
|
||||||
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
@ -14,19 +11,51 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
|
||||||
from openpyxl import load_workbook
|
from openpyxl import load_workbook, Workbook
|
||||||
import sys
|
import sys
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
from rag.nlp import find_codec
|
from rag.nlp import find_codec
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
|
||||||
class RAGFlowExcelParser:
|
class RAGFlowExcelParser:
|
||||||
def html(self, fnm, chunk_rows=256):
|
def html(self, fnm, chunk_rows=256):
|
||||||
if isinstance(fnm, str):
|
|
||||||
wb = load_workbook(fnm)
|
# if isinstance(fnm, str):
|
||||||
|
# wb = load_workbook(fnm)
|
||||||
|
# else:
|
||||||
|
# wb = load_workbook(BytesIO(fnm))++
|
||||||
|
|
||||||
|
s_fnm = fnm
|
||||||
|
if not isinstance(fnm, str):
|
||||||
|
s_fnm = BytesIO(fnm)
|
||||||
else:
|
else:
|
||||||
wb = load_workbook(BytesIO(fnm))
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
wb = load_workbook(s_fnm)
|
||||||
|
except Exception as e:
|
||||||
|
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
|
||||||
|
df = pd.read_excel(s_fnm)
|
||||||
|
wb = Workbook()
|
||||||
|
# if len(wb.worksheets) > 0:
|
||||||
|
# del wb.worksheets[0]
|
||||||
|
# else: pass
|
||||||
|
ws = wb.active
|
||||||
|
ws.title = "Data"
|
||||||
|
for col_num, column_name in enumerate(df.columns, 1):
|
||||||
|
ws.cell(row=1, column=col_num, value=column_name)
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
for row_num, row in enumerate(df.values, 2):
|
||||||
|
for col_num, value in enumerate(row, 1):
|
||||||
|
ws.cell(row=row_num, column=col_num, value=value)
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
|
||||||
tb_chunks = []
|
tb_chunks = []
|
||||||
for sheetname in wb.sheetnames:
|
for sheetname in wb.sheetnames:
|
||||||
@ -45,7 +74,7 @@ class RAGFlowExcelParser:
|
|||||||
tb += f"<table><caption>{sheetname}</caption>"
|
tb += f"<table><caption>{sheetname}</caption>"
|
||||||
tb += tb_rows_0
|
tb += tb_rows_0
|
||||||
for r in list(
|
for r in list(
|
||||||
rows[1 + chunk_i * chunk_rows : 1 + (chunk_i + 1) * chunk_rows]
|
rows[1 + chunk_i * chunk_rows: 1 + (chunk_i + 1) * chunk_rows]
|
||||||
):
|
):
|
||||||
tb += "<tr>"
|
tb += "<tr>"
|
||||||
for i, c in enumerate(r):
|
for i, c in enumerate(r):
|
||||||
@ -60,10 +89,41 @@ class RAGFlowExcelParser:
|
|||||||
return tb_chunks
|
return tb_chunks
|
||||||
|
|
||||||
def __call__(self, fnm):
|
def __call__(self, fnm):
|
||||||
if isinstance(fnm, str):
|
# if isinstance(fnm, str):
|
||||||
wb = load_workbook(fnm)
|
# wb = load_workbook(fnm)
|
||||||
|
# else:
|
||||||
|
# wb = load_workbook(BytesIO(fnm))
|
||||||
|
|
||||||
|
s_fnm = fnm
|
||||||
|
if not isinstance(fnm, str):
|
||||||
|
s_fnm = BytesIO(fnm)
|
||||||
else:
|
else:
|
||||||
wb = load_workbook(BytesIO(fnm))
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
wb = load_workbook(s_fnm)
|
||||||
|
except Exception as e:
|
||||||
|
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
|
||||||
|
df = pd.read_excel(s_fnm)
|
||||||
|
wb = Workbook()
|
||||||
|
if len(wb.worksheets) > 0:
|
||||||
|
del wb.worksheets[0]
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
ws = wb.active
|
||||||
|
ws.title = "Data"
|
||||||
|
for col_num, column_name in enumerate(df.columns, 1):
|
||||||
|
ws.cell(row=1, column=col_num, value=column_name)
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
for row_num, row in enumerate(df.values, 2):
|
||||||
|
for col_num, value in enumerate(row, 1):
|
||||||
|
ws.cell(row=row_num, column=col_num, value=value)
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
|
||||||
res = []
|
res = []
|
||||||
for sheetname in wb.sheetnames:
|
for sheetname in wb.sheetnames:
|
||||||
ws = wb[sheetname]
|
ws = wb[sheetname]
|
||||||
@ -104,3 +164,4 @@ class RAGFlowExcelParser:
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
psr = RAGFlowExcelParser()
|
psr = RAGFlowExcelParser()
|
||||||
psr(sys.argv[1])
|
psr(sys.argv[1])
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user