From f948c0d9f1fa2342583c051e2150839df851de20 Mon Sep 17 00:00:00 2001
From: Kevin Hu <kevinhu.sh@gmail.com>
Date: Fri, 27 Dec 2024 14:25:03 +0800
Subject: [PATCH] Clean query. (#4259)

### What problem does this PR solve?

#4239

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
---
 agent/component/__init__.py |  2 +-
 rag/nlp/query.py            | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/agent/component/__init__.py b/agent/component/__init__.py
index c124136b0..0d414fca9 100644
--- a/agent/component/__init__.py
+++ b/agent/component/__init__.py
@@ -36,12 +36,12 @@ from .iteration import Iteration, IterationParam
 from .iterationitem import IterationItem, IterationItemParam
 
 
-
 def component_class(class_name):
     m = importlib.import_module("agent.component")
     c = getattr(m, class_name)
     return c
 
+
 __all__ = [
     "Begin",
     "BeginParam",
diff --git a/rag/nlp/query.py b/rag/nlp/query.py
index 2d94a96c7..ab5c22e32 100644
--- a/rag/nlp/query.py
+++ b/rag/nlp/query.py
@@ -59,7 +59,7 @@ class FulltextQueryer:
                 "",
             ),
             (r"(^| )(what|who|how|which|where|why)('re|'s)? ", " "),
-            (r"(^| )('s|'re|is|are|were|was|do|does|did|don't|doesn't|didn't|has|have|be|there|you|me|your|my|mine|just|please|may|i|should|would|wouldn't|will|won't|done|go|for|with|so|the|a|an|by|i'm|it's|he's|she's|they|they're|you're|as|by|on|in|at|up|out|down|of) ", " ")
+            (r"(^| )('s|'re|is|are|were|was|do|does|did|don't|doesn't|didn't|has|have|be|there|you|me|your|my|mine|just|please|may|i|should|would|wouldn't|will|won't|done|go|for|with|so|the|a|an|by|i'm|it's|he's|she's|they|they're|you're|as|by|on|in|at|up|out|down|of|to|or|and|if) ", " ")
         ]
         for r, p in patts:
             txt = re.sub(r, p, txt, flags=re.IGNORECASE)
@@ -67,7 +67,7 @@ class FulltextQueryer:
 
     def question(self, txt, tbl="qa", min_match:float=0.6):
         txt = re.sub(
-            r"[ :\r\n\t,，。？?/`!！&\^%%()^\[\]]+",
+            r"[ :|\r\n\t,，。？?/`!！&^%%()\[\]{}<>]+",
             " ",
             rag_tokenizer.tradi2simp(rag_tokenizer.strQ2B(txt.lower())),
         ).strip()
@@ -81,16 +81,20 @@ class FulltextQueryer:
             tks_w = [(re.sub(r"[ \\\"'^]", "", tk), w) for tk, w in tks_w]
             tks_w = [(re.sub(r"^[a-z0-9]$", "", tk), w) for tk, w in tks_w if tk]
             tks_w = [(re.sub(r"^[\+-]", "", tk), w) for tk, w in tks_w if tk]
+            tks_w = [(tk.strip(), w) for tk, w in tks_w if tk.strip()]
             syns = []
             for tk, w in tks_w:
                 syn = self.syn.lookup(tk)
                 syn = rag_tokenizer.tokenize(" ".join(syn)).split()
                 keywords.extend(syn)
-                syn = ["\"{}\"^{:.4f}".format(s, w / 4.) for s in syn if s]
+                syn = ["\"{}\"^{:.4f}".format(s, w / 4.) for s in syn if s.strip()]
                 syns.append(" ".join(syn))
 
             q = ["({}^{:.4f}".format(tk, w) + " {})".format(syn) for (tk, w), syn in zip(tks_w, syns) if tk and not re.match(r"[.^+\(\)-]", tk)]
             for i in range(1, len(tks_w)):
+                left, right = tks_w[i - 1][0].strip(), tks_w[i][0].strip()
+                if not left or not right:
+                    continue
                 q.append(
                     '"%s %s"^%.4f'
                     % (