diff --git a/api/apps/dialog_app.py b/api/apps/dialog_app.py index 2969c1246..b59b02638 100644 --- a/api/apps/dialog_app.py +++ b/api/apps/dialog_app.py @@ -38,6 +38,7 @@ def set_dialog(): if not rerank_id: req["rerank_id"] = "" similarity_threshold = req.get("similarity_threshold", 0.1) vector_similarity_weight = req.get("vector_similarity_weight", 0.3) + if vector_similarity_weight is None: vector_similarity_weight = 0.3 llm_setting = req.get("llm_setting", {}) default_prompt = { "system": """你是一个智能助手,请总结知识库的内容来回答问题,请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时,你的回答必须包括“知识库中未找到您要的答案!”这句话。回答需要考虑聊天历史。 diff --git a/api/utils/file_utils.py b/api/utils/file_utils.py index a5af5edd3..ca5a16c43 100644 --- a/api/utils/file_utils.py +++ b/api/utils/file_utils.py @@ -156,7 +156,7 @@ def filename_type(filename): return FileType.PDF.value if re.match( - r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html)$", filename): + r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html)$", filename): return FileType.DOC.value if re.match( diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py index 9811d131e..ac2d3ef16 100644 --- a/deepdoc/parser/html_parser.py +++ b/deepdoc/parser/html_parser.py @@ -1,4 +1,16 @@ # -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from rag.nlp import find_codec import readability import html_text @@ -8,7 +20,7 @@ def get_encoding(file): with open(file,'rb') as f: tmp = chardet.detect(f.read()) return tmp['encoding'] - + class RAGFlowHtmlParser: def __call__(self, fnm, binary=None): txt = "" @@ -18,7 +30,7 @@ class RAGFlowHtmlParser: else: with open(fnm, "r",encoding=get_encoding(fnm)) as f: txt = f.read() - + html_doc = readability.Document(txt) title = html_doc.title() content = html_text.extract_text(html_doc.summary(html_partial=True)) diff --git a/rag/app/book.py b/rag/app/book.py index e0a1f9ffb..d969ccf92 100644 --- a/rag/app/book.py +++ b/rag/app/book.py @@ -135,7 +135,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, for ck in hierarchical_merge(bull, sections, 5)] else: sections = [s.split("@") for s, _ in sections] - sections = [(pr[0], "@" + pr[1]) for pr in sections if len(pr) == 2] + sections = [(pr[0], "@" + pr[1]) if len(pr) == 2 else (pr[0], '') for pr in sections ] chunks = naive_merge( sections, kwargs.get( "chunk_token_num", 256), kwargs.get(