AI/ragflow
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git synced 2025-08-12 11:59:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Clean query. (#4259)

Browse Source 
### What problem does this PR solve?

#4239

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Kevin Hu 2024-12-27 14:25:03 +08:00 committed by GitHub
parent c3e3f0fbb4
commit f948c0d9f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
agent/component/__init__.py
View File

@ -36,12 +36,12 @@ from .iteration import Iteration, IterationParam
from .iterationitem import IterationItem, IterationItemParam from .iterationitem import IterationItem, IterationItemParam
def component_class(class_name): def component_class(class_name):
m = importlib.import_module("agent.component") m = importlib.import_module("agent.component")
c = getattr(m, class_name) c = getattr(m, class_name)
return c return c
__all__ = [ __all__ = [
"Begin", "Begin",
"BeginParam", "BeginParam",

10
rag/nlp/query.py
View File

@ -59,7 +59,7 @@ class FulltextQueryer:
"", "",
), ),
(r"(^| )(what|who|how|which|where|why)('re|'s)? ", " "), (r"(^| )(what|who|how|which|where|why)('re|'s)? ", " "),
(r"(^| )('s|'re|is|are|were|was|do|does|did|don't|doesn't|didn't|has|have|be|there|you|me|your|my|mine|just|please|may|i|should|would|wouldn't|will|won't|done|go|for|with|so|the|a|an|by|i'm|it's|he's|she's|they|they're|you're|as|by|on|in|at|up|out|down|of) ", " ") (r"(^| )('s|'re|is|are|were|was|do|does|did|don't|doesn't|didn't|has|have|be|there|you|me|your|my|mine|just|please|may|i|should|would|wouldn't|will|won't|done|go|for|with|so|the|a|an|by|i'm|it's|he's|she's|they|they're|you're|as|by|on|in|at|up|out|down|of|to|or|and|if) ", " ")
] ]
for r, p in patts: for r, p in patts:
txt = re.sub(r, p, txt, flags=re.IGNORECASE) txt = re.sub(r, p, txt, flags=re.IGNORECASE)
@ -67,7 +67,7 @@ class FulltextQueryer:
def question(self, txt, tbl="qa", min_match:float=0.6): def question(self, txt, tbl="qa", min_match:float=0.6):
txt = re.sub( txt = re.sub(
r"[ :\r\n\t,，。？?/`!！&\^%%()^\[\]]+", r"[ :|\r\n\t,，。？?/`!！&^%%()\[\]{}<>]+",
" ", " ",
rag_tokenizer.tradi2simp(rag_tokenizer.strQ2B(txt.lower())), rag_tokenizer.tradi2simp(rag_tokenizer.strQ2B(txt.lower())),
).strip() ).strip()
@ -81,16 +81,20 @@ class FulltextQueryer:
tks_w = [(re.sub(r"[ \\\"'^]", "", tk), w) for tk, w in tks_w] tks_w = [(re.sub(r"[ \\\"'^]", "", tk), w) for tk, w in tks_w]
tks_w = [(re.sub(r"^[a-z0-9]$", "", tk), w) for tk, w in tks_w if tk] tks_w = [(re.sub(r"^[a-z0-9]$", "", tk), w) for tk, w in tks_w if tk]
tks_w = [(re.sub(r"^[\+-]", "", tk), w) for tk, w in tks_w if tk] tks_w = [(re.sub(r"^[\+-]", "", tk), w) for tk, w in tks_w if tk]
tks_w = [(tk.strip(), w) for tk, w in tks_w if tk.strip()]
syns = [] syns = []
for tk, w in tks_w: for tk, w in tks_w:
syn = self.syn.lookup(tk) syn = self.syn.lookup(tk)
syn = rag_tokenizer.tokenize(" ".join(syn)).split() syn = rag_tokenizer.tokenize(" ".join(syn)).split()
keywords.extend(syn) keywords.extend(syn)
syn = ["\"{}\"^{:.4f}".format(s, w / 4.) for s in syn if s] syn = ["\"{}\"^{:.4f}".format(s, w / 4.) for s in syn if s.strip()]
syns.append(" ".join(syn)) syns.append(" ".join(syn))
q = ["({}^{:.4f}".format(tk, w) + " {})".format(syn) for (tk, w), syn in zip(tks_w, syns) if tk and not re.match(r"[.^+\(\)-]", tk)] q = ["({}^{:.4f}".format(tk, w) + " {})".format(syn) for (tk, w), syn in zip(tks_w, syns) if tk and not re.match(r"[.^+\(\)-]", tk)]
for i in range(1, len(tks_w)): for i in range(1, len(tks_w)):
left, right = tks_w[i - 1][0].strip(), tks_w[i][0].strip()
if not left or not right:
continue
q.append( q.append(
'"%s %s"^%.4f' '"%s %s"^%.4f'
% ( % (