From 289034f36ecbfa3757ad1e4225a762c81266e7c1 Mon Sep 17 00:00:00 2001 From: Kevin Hu Date: Wed, 20 Nov 2024 11:13:12 +0800 Subject: [PATCH] smooth term weight (#3510) ### What problem does this PR solve? #3499 ### Type of change - [x] Performance Improvement --- rag/nlp/term_weight.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.py index 8e1e59836..374065a0c 100644 --- a/rag/nlp/term_weight.py +++ b/rag/nlp/term_weight.py @@ -228,7 +228,7 @@ class Dealer: idf2 = np.array([idf(df(t), 1000000000) for t in tks]) wts = (0.3 * idf1 + 0.7 * idf2) * \ np.array([ner(t) * postag(t) for t in tks]) - wts = [math.exp(s) for s in wts] + wts = [math.pow(s, 2) for s in wts] tw = list(zip(tks, wts)) else: for tk in tks: @@ -237,7 +237,7 @@ class Dealer: idf2 = np.array([idf(df(t), 1000000000) for t in tt]) wts = (0.3 * idf1 + 0.7 * idf2) * \ np.array([ner(t) * postag(t) for t in tt]) - wts = [math.exp(s) for s in wts] + wts = [math.pow(s, 2) for s in wts] tw.extend(zip(tt, wts)) S = np.sum([s for _, s in tw])