mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-05-16 07:58:15 +08:00
81 lines
2.8 KiB
Python
81 lines
2.8 KiB
Python
import re,json,os
|
||
import pandas as pd
|
||
from rag.nlp import huqie
|
||
from . import regions
|
||
current_file_path = os.path.dirname(os.path.abspath(__file__))
|
||
GOODS = pd.read_csv(os.path.join(current_file_path, "res/corp_baike_len.csv"), sep="\t", header=0).fillna(0)
|
||
GOODS["cid"] = GOODS["cid"].astype(str)
|
||
GOODS = GOODS.set_index(["cid"])
|
||
CORP_TKS = json.load(open(os.path.join(current_file_path, "res/corp.tks.freq.json"), "r"))
|
||
GOOD_CORP = json.load(open(os.path.join(current_file_path, "res/good_corp.json"), "r"))
|
||
CORP_TAG = json.load(open(os.path.join(current_file_path, "res/corp_tag.json"), "r"))
|
||
|
||
def baike(cid, default_v=0):
|
||
global GOODS
|
||
try:
|
||
return GOODS.loc[str(cid), "len"]
|
||
except Exception as e:
|
||
pass
|
||
return default_v
|
||
|
||
|
||
def corpNorm(nm, add_region=True):
|
||
global CORP_TKS
|
||
if not nm or type(nm)!=type(""):return ""
|
||
nm = huqie.tradi2simp(huqie.strQ2B(nm)).lower()
|
||
nm = re.sub(r"&", "&", nm)
|
||
nm = re.sub(r"[\(\)（）\+'\"\t \*\\【】-]+", " ", nm)
|
||
nm = re.sub(r"([—-]+.*| +co\..*|corp\..*| +inc\..*| +ltd.*)", "", nm, 10000, re.IGNORECASE)
|
||
nm = re.sub(r"(计算机|技术|(技术|科技|网络)*有限公司|公司|有限|研发中心|中国|总部)$", "", nm, 10000, re.IGNORECASE)
|
||
if not nm or (len(nm)<5 and not regions.isName(nm[0:2])):return nm
|
||
|
||
tks = huqie.qie(nm).split(" ")
|
||
reg = [t for i,t in enumerate(tks) if regions.isName(t) and (t != "中国" or i > 0)]
|
||
nm = ""
|
||
for t in tks:
|
||
if regions.isName(t) or t in CORP_TKS:continue
|
||
if re.match(r"[0-9a-zA-Z\\,.]+", t) and re.match(r".*[0-9a-zA-Z\,.]+$", nm):nm += " "
|
||
nm += t
|
||
|
||
r = re.search(r"^([^a-z0-9 \(\)&]{2,})[a-z ]{4,}$", nm.strip())
|
||
if r:nm = r.group(1)
|
||
r = re.search(r"^([a-z ]{3,})[^a-z0-9 \(\)&]{2,}$", nm.strip())
|
||
if r:nm = r.group(1)
|
||
return nm.strip() + (("" if not reg else "(%s)"%reg[0]) if add_region else "")
|
||
|
||
|
||
def rmNoise(n):
|
||
n = re.sub(r"[\(（][^()（）]+[)）]", "", n)
|
||
n = re.sub(r"[,. &（）()]+", "", n)
|
||
return n
|
||
|
||
GOOD_CORP = set([corpNorm(rmNoise(c), False) for c in GOOD_CORP])
|
||
for c,v in CORP_TAG.items():
|
||
cc = corpNorm(rmNoise(c), False)
|
||
if not cc: print (c)
|
||
CORP_TAG = {corpNorm(rmNoise(c), False):v for c,v in CORP_TAG.items()}
|
||
|
||
def is_good(nm):
|
||
global GOOD_CORP
|
||
if nm.find("外派")>=0:return False
|
||
nm = rmNoise(nm)
|
||
nm = corpNorm(nm, False)
|
||
for n in GOOD_CORP:
|
||
if re.match(r"[0-9a-zA-Z]+$", n):
|
||
if n == nm: return True
|
||
elif nm.find(n)>=0:return True
|
||
return False
|
||
|
||
def corp_tag(nm):
|
||
global CORP_TAG
|
||
nm = rmNoise(nm)
|
||
nm = corpNorm(nm, False)
|
||
for n in CORP_TAG.keys():
|
||
if re.match(r"[0-9a-zA-Z., ]+$", n):
|
||
if n == nm: return CORP_TAG[n]
|
||
elif nm.find(n)>=0:
|
||
if len(n)<3 and len(nm)/len(n)>=2:continue
|
||
return CORP_TAG[n]
|
||
return []
|
||
|