diff --git a/api/utils/file_utils.py b/api/utils/file_utils.py index ab859aaba..a5af5edd3 100644 --- a/api/utils/file_utils.py +++ b/api/utils/file_utils.py @@ -156,7 +156,7 @@ def filename_type(filename): return FileType.PDF.value if re.match( - r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt)$", filename): + r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html)$", filename): return FileType.DOC.value if re.match( diff --git a/deepdoc/parser/__init__.py b/deepdoc/parser/__init__.py index b9de7eab7..99ba946f9 100644 --- a/deepdoc/parser/__init__.py +++ b/deepdoc/parser/__init__.py @@ -4,3 +4,4 @@ from .pdf_parser import RAGFlowPdfParser as PdfParser, PlainParser from .docx_parser import RAGFlowDocxParser as DocxParser from .excel_parser import RAGFlowExcelParser as ExcelParser from .ppt_parser import RAGFlowPptParser as PptParser +from .html_parser import RAGFlowHtmlParser as HtmlParser diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py new file mode 100644 index 000000000..9811d131e --- /dev/null +++ b/deepdoc/parser/html_parser.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from rag.nlp import find_codec +import readability +import html_text +import chardet + +def get_encoding(file): + with open(file,'rb') as f: + tmp = chardet.detect(f.read()) + return tmp['encoding'] + +class RAGFlowHtmlParser: + def __call__(self, fnm, binary=None): + txt = "" + if binary: + encoding = find_codec(binary) + txt = binary.decode(encoding, errors="ignore") + else: + with open(fnm, "r",encoding=get_encoding(fnm)) as f: + txt = f.read() + + html_doc = readability.Document(txt) + title = html_doc.title() + content = html_text.extract_text(html_doc.summary(html_partial=True)) + txt = f'{title}\n{content}' + sections = txt.split("\n") + return sections diff --git a/rag/app/book.py b/rag/app/book.py index c4bc62abf..e0a1f9ffb 100644 --- a/rag/app/book.py +++ b/rag/app/book.py @@ -19,7 +19,7 @@ from rag.nlp import bullets_category, is_english, tokenize, remove_contents_tabl hierarchical_merge, make_colon_as_title, naive_merge, random_choices, tokenize_table, add_positions, \ tokenize_chunks, find_codec from rag.nlp import rag_tokenizer -from deepdoc.parser import PdfParser, DocxParser, PlainParser +from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser class Pdf(PdfParser): @@ -105,6 +105,14 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, random_choices([t for t, _ in sections], k=200))) callback(0.8, "Finish parsing.") + elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE): + callback(0.1, "Start to parse.") + sections = HtmlParser()(filename, binary) + sections = [(l, "") for l in sections if l] + remove_contents_table(sections, eng=is_english( + random_choices([t for t, _ in sections], k=200))) + callback(0.8, "Finish parsing.") + elif re.search(r"\.doc$", filename, re.IGNORECASE): callback(0.1, "Start to parse.") binary = BytesIO(binary) diff --git a/rag/app/laws.py b/rag/app/laws.py index 6361d62cb..dd8060670 100644 --- a/rag/app/laws.py +++ b/rag/app/laws.py @@ -20,7 +20,7 @@ from api.db import ParserType from rag.nlp import bullets_category, is_english, tokenize, remove_contents_table, hierarchical_merge, \ make_colon_as_title, add_positions, tokenize_chunks, find_codec from rag.nlp import rag_tokenizer -from deepdoc.parser import PdfParser, DocxParser, PlainParser +from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser from rag.settings import cron_logger @@ -125,6 +125,12 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, sections = [l for l in sections if l] callback(0.8, "Finish parsing.") + elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE): + callback(0.1, "Start to parse.") + sections = HtmlParser()(filename, binary) + sections = [l for l in sections if l] + callback(0.8, "Finish parsing.") + elif re.search(r"\.doc$", filename, re.IGNORECASE): callback(0.1, "Start to parse.") binary = BytesIO(binary) diff --git a/rag/app/naive.py b/rag/app/naive.py index f1e3e3bdb..0f4bd2434 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -17,7 +17,7 @@ from timeit import default_timer as timer import re from deepdoc.parser.pdf_parser import PlainParser from rag.nlp import rag_tokenizer, naive_merge, tokenize_table, tokenize_chunks, find_codec -from deepdoc.parser import PdfParser, ExcelParser, DocxParser +from deepdoc.parser import PdfParser, ExcelParser, DocxParser, HtmlParser from rag.settings import cron_logger from rag.utils import num_tokens_from_string @@ -161,6 +161,12 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, callback(0.8, "Finish parsing.") + elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE): + callback(0.1, "Start to parse.") + sections = HtmlParser()(filename, binary) + sections = [(l, "") for l in sections if l] + callback(0.8, "Finish parsing.") + elif re.search(r"\.doc$", filename, re.IGNORECASE): callback(0.1, "Start to parse.") binary = BytesIO(binary) diff --git a/rag/app/one.py b/rag/app/one.py index 80339513b..c0d132447 100644 --- a/rag/app/one.py +++ b/rag/app/one.py @@ -15,7 +15,7 @@ from io import BytesIO import re from rag.app import laws from rag.nlp import rag_tokenizer, tokenize, find_codec -from deepdoc.parser import PdfParser, ExcelParser, PlainParser +from deepdoc.parser import PdfParser, ExcelParser, PlainParser, HtmlParser class Pdf(PdfParser): @@ -97,6 +97,12 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, sections = [s for s in sections if s] callback(0.8, "Finish parsing.") + elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE): + callback(0.1, "Start to parse.") + sections = HtmlParser()(filename, binary) + sections = [s for s in sections if s] + callback(0.8, "Finish parsing.") + elif re.search(r"\.doc$", filename, re.IGNORECASE): callback(0.1, "Start to parse.") binary = BytesIO(binary) diff --git a/requirements.txt b/requirements.txt index c8c3cd25d..08e54fc24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -137,3 +137,5 @@ loguru==0.7.2 umap-learn fasttext==0.9.2 volcengine +readability-lxml==0.8.1 +html_text==0.6.2 \ No newline at end of file diff --git a/requirements_arm.txt b/requirements_arm.txt index d03ffbaef..7c731263c 100644 --- a/requirements_arm.txt +++ b/requirements_arm.txt @@ -137,4 +137,6 @@ loguru==0.7.2 umap-learn fasttext==0.9.2 volcengine -opencv-python-headless==4.9.0.80 \ No newline at end of file +opencv-python-headless==4.9.0.80 +readability-lxml==0.8.1 +html_text==0.6.2 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index ae0ba7a34..e5419b56a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -125,3 +125,5 @@ redis==5.0.4 fasttext==0.9.2 umap-learn volcengine +readability-lxml==0.8.1 +html_text==0.6.2 \ No newline at end of file