AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-13 17:28:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

make titles in markdown not be splited with following content (#2971)

Browse Source 
### What problem does this PR solve?

#2970 
### Type of change

- [ ] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Kevin Hu 2024-10-22 15:25:23 +08:00 committed by GitHub
parent adb0a93d95
commit 1fce6caf80
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
rag/app/manual.py
View File

@ -67,9 +67,11 @@ class Pdf(PdfParser):
return [(b["text"], b.get("layout_no", ""), self.get_position(b, zoomin)) return [(b["text"], b.get("layout_no", ""), self.get_position(b, zoomin))
for i, b in enumerate(self.boxes)], tbls for i, b in enumerate(self.boxes)], tbls
class Docx(DocxParser): class Docx(DocxParser):
def __init__(self): def __init__(self):
pass pass
def get_picture(self, document, paragraph): def get_picture(self, document, paragraph):
img = paragraph._element.xpath('.//pic:pic') img = paragraph._element.xpath('.//pic:pic')
if not img: if not img:
@ -80,6 +82,7 @@ class Docx(DocxParser):
image = related_part.image image = related_part.image
image = Image.open(BytesIO(image.blob)) image = Image.open(BytesIO(image.blob))
return image return image
def concat_img(self, img1, img2): def concat_img(self, img1, img2):
if img1 and not img2: if img1 and not img2:
return img1 return img1
@ -160,6 +163,7 @@ class Docx(DocxParser):
tbls.append(((None, html), "")) tbls.append(((None, html), ""))
return ti_list, tbls return ti_list, tbls
def chunk(filename, binary=None, from_page=0, to_page=100000, def chunk(filename, binary=None, from_page=0, to_page=100000,
lang="Chinese", callback=None, **kwargs): lang="Chinese", callback=None, **kwargs):
""" """
@ -244,6 +248,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
res = tokenize_table(tbls, doc, eng) res = tokenize_table(tbls, doc, eng)
res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser)) res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
return res return res
if re.search(r"\.docx$", filename, re.IGNORECASE): if re.search(r"\.docx$", filename, re.IGNORECASE):
docx_parser = Docx() docx_parser = Docx()
ti_list, tbls = docx_parser(filename, binary, ti_list, tbls = docx_parser(filename, binary,
@ -259,8 +264,6 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
raise NotImplementedError("file type not supported yet(pdf and docx supported)") raise NotImplementedError("file type not supported yet(pdf and docx supported)")
if __name__ == "__main__": if __name__ == "__main__":
import sys import sys

6
rag/app/naive.py
View File

@ -167,9 +167,13 @@ class Markdown(MarkdownParser):
if num_tokens_from_string(sec) > 10 * self.chunk_token_num: if num_tokens_from_string(sec) > 10 * self.chunk_token_num:
sections.append((sec[:int(len(sec) / 2)], "")) sections.append((sec[:int(len(sec) / 2)], ""))
sections.append((sec[int(len(sec) / 2):], "")) sections.append((sec[int(len(sec) / 2):], ""))
else:
if sections and sections[-1][0].strip().find("#") == 0:
sec_, _ = sections.pop(-1)
sections.append((sec_+"\n"+sec, ""))
else: else:
sections.append((sec, "")) sections.append((sec, ""))
print(tables)
for table in tables: for table in tables:
tbls.append(((None, markdown(table, extensions=['markdown.extensions.tables'])), "")) tbls.append(((None, markdown(table, extensions=['markdown.extensions.tables'])), ""))
return sections, tbls return sections, tbls

1
rag/app/qa.py
View File

@ -393,6 +393,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
if sum_question: if sum_question:
res.append(beAdoc(deepcopy(doc), sum_question, markdown(last_answer, extensions=['markdown.extensions.tables']), eng)) res.append(beAdoc(deepcopy(doc), sum_question, markdown(last_answer, extensions=['markdown.extensions.tables']), eng))
return res return res
elif re.search(r"\.docx$", filename, re.IGNORECASE): elif re.search(r"\.docx$", filename, re.IGNORECASE):
docx_parser = Docx() docx_parser = Docx()
qai_list, tbls = docx_parser(filename, binary, qai_list, tbls = docx_parser(filename, binary,