mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-07-24 17:04:29 +08:00
refine manual parser (#140)
This commit is contained in:
KevinHuSh
GitHub
parent
f4ec7cfa76
commit
6c6b144de2
@ -118,14 +118,13 @@ def message_fit_in(msg, max_length=4000):
|
||||
|
||||
c = count()
|
||||
if c < max_length: return c, msg
|
||||
msg = [m for m in msg if m.role in ["system", "user"]]
|
||||
c = count()
|
||||
if c < max_length: return c, msg
|
||||
|
||||
msg_ = [m for m in msg[:-1] if m.role == "system"]
|
||||
msg_.append(msg[-1])
|
||||
msg = msg_
|
||||
c = count()
|
||||
if c < max_length: return c, msg
|
||||
|
||||
ll = num_tokens_from_string(msg_[0].content)
|
||||
l = num_tokens_from_string(msg_[-1].content)
|
||||
if ll / (ll + l) > 0.8:
|
||||
|
||||
@ -218,7 +218,7 @@ def rm():
|
||||
ELASTICSEARCH.deleteByQuery(Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
|
||||
|
||||
DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0)
|
||||
if not DocumentService.delete_by_id(req["doc_id"]):
|
||||
if not DocumentService.delete(doc):
|
||||
return get_data_error_result(
|
||||
retmsg="Database error (Document removal)!")
|
||||
|
||||
|
||||
@ -353,7 +353,7 @@ class User(DataBaseModel, UserMixin):
|
||||
email = CharField(max_length=255, null=False, help_text="email", index=True)
|
||||
avatar = TextField(null=True, help_text="avatar base64 string")
|
||||
language = CharField(max_length=32, null=True, help_text="English|Chinese", default="Chinese")
|
||||
color_schema = CharField(max_length=32, null=True, help_text="Bright|Dark", default="Dark")
|
||||
color_schema = CharField(max_length=32, null=True, help_text="Bright|Dark", default="Bright")
|
||||
timezone = CharField(max_length=64, null=True, help_text="Timezone", default="UTC+8\tAsia/Shanghai")
|
||||
last_login_time = DateTimeField(null=True)
|
||||
is_authenticated = CharField(max_length=1, null=False, default="1")
|
||||
|
||||
@ -223,7 +223,7 @@ def init_llm_factory():
|
||||
"fid": factory_infos[3]["name"],
|
||||
"llm_name": "qwen-14B-chat",
|
||||
"tags": "LLM,CHAT,",
|
||||
"max_tokens": 8191,
|
||||
"max_tokens": 4096,
|
||||
"model_type": LLMType.CHAT.value
|
||||
}, {
|
||||
"fid": factory_infos[3]["name"],
|
||||
@ -271,11 +271,15 @@ def init_llm_factory():
|
||||
pass
|
||||
|
||||
"""
|
||||
modify service_config
|
||||
drop table llm;
|
||||
drop table factories;
|
||||
drop table llm_factories;
|
||||
update tenant_llm set llm_factory='Tongyi-Qianwen' where llm_factory='通义千问';
|
||||
update tenant_llm set llm_factory='ZHIPU-AI' where llm_factory='智谱AI';
|
||||
update tenant set parser_ids='naive:General,one:One,qa:Q&A,resume:Resume,table:Table,laws:Laws,manual:Manual,book:Book,paper:Paper,presentation:Presentation,picture:Picture';
|
||||
alter table knowledgebase modify avatar longtext;
|
||||
alter table user modify avatar longtext;
|
||||
alter table dialog modify icon longtext;
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@ -60,6 +60,15 @@ class DocumentService(CommonService):
|
||||
raise RuntimeError("Database error (Knowledgebase)!")
|
||||
return doc
|
||||
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def delete(cls, doc):
|
||||
e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
|
||||
if not KnowledgebaseService.update_by_id(
|
||||
kb.id, {"doc_num": kb.doc_num - 1}):
|
||||
raise RuntimeError("Database error (Knowledgebase)!")
|
||||
return cls.delete_by_id(doc.id)
|
||||
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def get_newly_uploaded(cls, tm, mod=0, comm=1, items_per_page=64):
|
||||
|
||||
@ -11,7 +11,7 @@ import logging
|
||||
from PIL import Image, ImageDraw
|
||||
import numpy as np
|
||||
|
||||
from api.db import ParserType
|
||||
from PyPDF2 import PdfReader as pdf2_read
|
||||
from deepdoc.vision import OCR, Recognizer, LayoutRecognizer, TableStructureRecognizer
|
||||
from rag.nlp import huqie
|
||||
from copy import deepcopy
|
||||
@ -288,9 +288,9 @@ class HuParser:
|
||||
for b in bxs])
|
||||
self.boxes.append(bxs)
|
||||
|
||||
def _layouts_rec(self, ZM):
|
||||
def _layouts_rec(self, ZM, drop=True):
|
||||
assert len(self.page_images) == len(self.boxes)
|
||||
self.boxes, self.page_layout = self.layouter(self.page_images, self.boxes, ZM)
|
||||
self.boxes, self.page_layout = self.layouter(self.page_images, self.boxes, ZM, drop=drop)
|
||||
# cumlative Y
|
||||
for i in range(len(self.boxes)):
|
||||
self.boxes[i]["top"] += \
|
||||
@ -908,6 +908,23 @@ class HuParser:
|
||||
self.page_images.append(img)
|
||||
self.page_chars.append([])
|
||||
|
||||
self.outlines = []
|
||||
try:
|
||||
self.pdf = pdf2_read(fnm if isinstance(fnm, str) else BytesIO(fnm))
|
||||
outlines = self.pdf.outline
|
||||
|
||||
def dfs(arr, depth):
|
||||
for a in arr:
|
||||
if isinstance(a, dict):
|
||||
self.outlines.append((a["/Title"], depth))
|
||||
continue
|
||||
dfs(a, depth+1)
|
||||
dfs(outlines, 0)
|
||||
except Exception as e:
|
||||
logging.warning(f"Outlines exception: {e}")
|
||||
if not self.outlines:
|
||||
logging.warning(f"Miss outlines")
|
||||
|
||||
logging.info("Images converted.")
|
||||
self.is_english = [re.search(r"[a-zA-Z0-9,/¸;:'\[\]\(\)!@#$%^&*\"?<>._-]{30,}", "".join(
|
||||
random.choices([c["text"] for c in self.page_chars[i]], k=min(100, len(self.page_chars[i]))))) for i in
|
||||
|
||||
@ -39,7 +39,7 @@ class LayoutRecognizer(Recognizer):
|
||||
super().__init__(self.labels, domain, os.path.join(get_project_base_directory(), "rag/res/deepdoc/"))
|
||||
self.garbage_layouts = ["footer", "header", "reference"]
|
||||
|
||||
def __call__(self, image_list, ocr_res, scale_factor=3, thr=0.2, batch_size=16):
|
||||
def __call__(self, image_list, ocr_res, scale_factor=3, thr=0.2, batch_size=16, drop=True):
|
||||
def __is_garbage(b):
|
||||
patt = [r"^•+$", r"(版权归©|免责条款|地址[:：])", r"\.{3,}", "^[0-9]{1,2} / ?[0-9]{1,2}$",
|
||||
r"^[0-9]{1,2} of [0-9]{1,2}$", "^http://[^ ]{12,}",
|
||||
@ -88,7 +88,11 @@ class LayoutRecognizer(Recognizer):
|
||||
i += 1
|
||||
continue
|
||||
lts_[ii]["visited"] = True
|
||||
if lts_[ii]["type"] in self.garbage_layouts:
|
||||
keep_feats = [
|
||||
lts_[ii]["type"] == "footer" and bxs[i]["bottom"] < image_list[pn].size[1]*0.9/scale_factor,
|
||||
lts_[ii]["type"] == "header" and bxs[i]["top"] > image_list[pn].size[1]*0.1/scale_factor,
|
||||
]
|
||||
if drop and lts_[ii]["type"] in self.garbage_layouts and not any(keep_feats):
|
||||
if lts_[ii]["type"] not in garbages:
|
||||
garbages[lts_[ii]["type"]] = []
|
||||
garbages[lts_[ii]["type"]].append(bxs[i]["text"])
|
||||
|
||||
@ -51,15 +51,30 @@ class Pdf(PdfParser):
|
||||
|
||||
# set pivot using the most frequent type of title,
|
||||
# then merge between 2 pivot
|
||||
bull = bullets_category([b["text"] for b in self.boxes])
|
||||
most_level, levels = title_frequency(bull, [(b["text"], b.get("layout_no","")) for b in self.boxes])
|
||||
if len(self.boxes)>0 and len(self.outlines)/len(self.boxes) > 0.1:
|
||||
max_lvl = max([lvl for _, lvl in self.outlines])
|
||||
most_level = max(0, max_lvl-1)
|
||||
levels = []
|
||||
for b in self.boxes:
|
||||
for t,lvl in self.outlines:
|
||||
tks = set([t[i]+t[i+1] for i in range(len(t)-1)])
|
||||
tks_ = set([b["text"][i]+b["text"][i+1] for i in range(min(len(t), len(b["text"])-1))])
|
||||
if len(set(tks & tks_))/max([len(tks), len(tks_), 1]) > 0.8:
|
||||
levels.append(lvl)
|
||||
break
|
||||
else:
|
||||
levels.append(max_lvl + 1)
|
||||
else:
|
||||
bull = bullets_category([b["text"] for b in self.boxes])
|
||||
most_level, levels = title_frequency(bull, [(b["text"], b.get("layout_no","")) for b in self.boxes])
|
||||
|
||||
assert len(self.boxes) == len(levels)
|
||||
sec_ids = []
|
||||
sid = 0
|
||||
for i, lvl in enumerate(levels):
|
||||
if lvl <= most_level and i > 0 and lvl != levels[i-1]: sid += 1
|
||||
sec_ids.append(sid)
|
||||
#print(lvl, self.boxes[i]["text"], most_level)
|
||||
#print(lvl, self.boxes[i]["text"], most_level, sid)
|
||||
|
||||
sections = [(b["text"], sec_ids[i], self.get_position(b, zoomin)) for i, b in enumerate(self.boxes)]
|
||||
for (img, rows), poss in tbls:
|
||||
@ -67,13 +82,16 @@ class Pdf(PdfParser):
|
||||
|
||||
chunks = []
|
||||
last_sid = -2
|
||||
tk_cnt = 0
|
||||
for txt, sec_id, poss in sorted(sections, key=lambda x: (x[-1][0][0], x[-1][0][3], x[-1][0][1])):
|
||||
poss = "\t".join([tag(*pos) for pos in poss])
|
||||
if sec_id == last_sid or sec_id == -1:
|
||||
if tk_cnt < 2048 and (sec_id == last_sid or sec_id == -1):
|
||||
if chunks:
|
||||
chunks[-1] += "\n" + txt + poss
|
||||
tk_cnt += num_tokens_from_string(txt)
|
||||
continue
|
||||
chunks.append(txt + poss)
|
||||
tk_cnt = num_tokens_from_string(txt)
|
||||
if sec_id >-1: last_sid = sec_id
|
||||
return chunks, tbls
|
||||
|
||||
@ -97,37 +115,17 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, lang="Chinese", ca
|
||||
# is it English
|
||||
eng = lang.lower() == "english"#pdf_parser.is_english
|
||||
|
||||
i = 0
|
||||
chunk = []
|
||||
tk_cnt = 0
|
||||
res = tokenize_table(tbls, doc, eng)
|
||||
def add_chunk():
|
||||
nonlocal chunk, res, doc, pdf_parser, tk_cnt
|
||||
for ck in cks:
|
||||
d = copy.deepcopy(doc)
|
||||
ck = "\n".join(chunk)
|
||||
tokenize(d, pdf_parser.remove_tag(ck), eng)
|
||||
d["image"], poss = pdf_parser.crop(ck, need_position=True)
|
||||
add_positions(d, poss)
|
||||
tokenize(d, pdf_parser.remove_tag(ck), eng)
|
||||
res.append(d)
|
||||
chunk = []
|
||||
tk_cnt = 0
|
||||
|
||||
while i < len(cks):
|
||||
if tk_cnt > 256: add_chunk()
|
||||
txt = cks[i]
|
||||
txt_ = pdf_parser.remove_tag(txt)
|
||||
i += 1
|
||||
cnt = num_tokens_from_string(txt_)
|
||||
chunk.append(txt)
|
||||
tk_cnt += cnt
|
||||
if chunk: add_chunk()
|
||||
|
||||
for i, d in enumerate(res):
|
||||
print(d)
|
||||
# d["image"].save(f"./logs/{i}.jpg")
|
||||
return res
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
def dummy(prog=None, msg=""):
|
||||
|
||||
@ -10,12 +10,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import copy
|
||||
import re
|
||||
from rag.app import laws
|
||||
from rag.nlp import huqie, is_english, tokenize, naive_merge, tokenize_table, add_positions
|
||||
from rag.nlp import huqie, tokenize
|
||||
from deepdoc.parser import PdfParser, ExcelParser
|
||||
from rag.settings import cron_logger
|
||||
|
||||
|
||||
class Pdf(PdfParser):
|
||||
@ -33,7 +31,7 @@ class Pdf(PdfParser):
|
||||
|
||||
from timeit import default_timer as timer
|
||||
start = timer()
|
||||
self._layouts_rec(zoomin)
|
||||
self._layouts_rec(zoomin, drop=False)
|
||||
callback(0.63, "Layout analysis finished.")
|
||||
print("paddle layouts:", timer() - start)
|
||||
self._table_transformer_job(zoomin)
|
||||
|
||||
@ -215,7 +215,7 @@ class Dealer:
|
||||
else:
|
||||
pieces = re.split(r"([^\|][；。？!！\n]|[a-z][.?;!][ \n])", answer)
|
||||
for i in range(1, len(pieces)):
|
||||
if re.match(r"[a-z][.?;!][ \n]", pieces[i]):
|
||||
if re.match(r"([^\|][；。？!！\n]|[a-z][.?;!][ \n])", pieces[i]):
|
||||
pieces[i - 1] += pieces[i][0]
|
||||
pieces[i] = pieces[i][1:]
|
||||
idx = []
|
||||
@ -243,7 +243,8 @@ class Dealer:
|
||||
chunks_tks,
|
||||
tkweight, vtweight)
|
||||
mx = np.max(sim) * 0.99
|
||||
if mx < 0.65:
|
||||
es_logger.info("{} SIM: {}".format(pieces_[i], mx))
|
||||
if mx < 0.63:
|
||||
continue
|
||||
cites[idx[i]] = list(
|
||||
set([str(ii) for ii in range(len(chunk_v)) if sim[ii] > mx]))[:4]
|
||||
|
||||
@ -82,8 +82,8 @@ def dispatch():
|
||||
tsks = []
|
||||
if r["type"] == FileType.PDF.value:
|
||||
pages = PdfParser.total_page_number(r["name"], MINIO.get(r["kb_id"], r["location"]))
|
||||
page_size = 5
|
||||
if r["parser_id"] == "paper": page_size = 12
|
||||
page_size = 12
|
||||
if r["parser_id"] == "paper": page_size = 22
|
||||
if r["parser_id"] == "one": page_size = 1000000000
|
||||
for s,e in r["parser_config"].get("pages", [(0,100000)]):
|
||||
e = min(e, pages)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user