mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-04-24 07:00:02 +08:00
Add Graph Baidusearch and dsl_example (#1378)
### What problem does this PR solve? #918 ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
GYH
GitHub
parent
957cd55e4a
commit
cf542e80b3
@ -8,6 +8,8 @@ from .switch import Switch, SwitchParam
|
||||
from .relevant import Relevant, RelevantParam
|
||||
from .message import Message, MessageParam
|
||||
from .rewrite import RewriteQuestion, RewriteQuestionParam
|
||||
from .keyword import KeywordExtract, KeywordExtractParam
|
||||
from .baidu import Baidu, BaiduParam
|
||||
|
||||
|
||||
def component_class(class_name):
|
||||
|
||||
62
graph/component/baidu.py
Normal file
62
graph/component/baidu.py
Normal file
@ -0,0 +1,62 @@
|
||||
#
|
||||
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import random
|
||||
from abc import ABC
|
||||
from functools import partial
|
||||
import pandas as pd
|
||||
import requests
|
||||
import re
|
||||
|
||||
from graph.component.base import ComponentBase, ComponentParamBase
|
||||
|
||||
|
||||
class BaiduParam(ComponentParamBase):
|
||||
"""
|
||||
Define the Baidu component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.top_n = 10
|
||||
|
||||
def check(self):
|
||||
self.check_positive_integer(self.top_n, "Top N")
|
||||
|
||||
|
||||
class Baidu(ComponentBase, ABC):
|
||||
component_name = "Baidu"
|
||||
|
||||
def _run(self, history, **kwargs):
|
||||
ans = self.get_input()
|
||||
ans = " - ".join(ans["content"]) if "content" in ans else ""
|
||||
if not ans:
|
||||
return Baidu.be_output(self._param.no)
|
||||
|
||||
url = 'https://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n)
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'}
|
||||
response = requests.get(url=url, headers=headers)
|
||||
|
||||
baidu_res = re.findall(r'"contentText":"(.*?)"', response.text)
|
||||
url_res = re.findall(r"'url': \\\"(.*?)\\\"}", response.text)
|
||||
for i in range(min(len(baidu_res), len(url_res))):
|
||||
baidu_res[i] += '<a>' + url_res[i] + '</a>'
|
||||
|
||||
del url_res
|
||||
|
||||
br = pd.DataFrame(baidu_res, columns=['content'])
|
||||
print(">>>>>>>>>>>>>>>>>>>>>>>>>>\n", br)
|
||||
return br
|
||||
@ -22,18 +22,17 @@ from graph.settings import DEBUG
|
||||
|
||||
|
||||
class KeywordExtractParam(GenerateParam):
|
||||
|
||||
"""
|
||||
Define the KeywordExtract component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.temperature = 0.5
|
||||
self.prompt = ""
|
||||
self.topn = 1
|
||||
self.top_n = 1
|
||||
|
||||
def check(self):
|
||||
super().check()
|
||||
self.check_positive_integer(self.top_n, "Top N")
|
||||
|
||||
def get_prompt(self):
|
||||
self.prompt = """
|
||||
@ -43,12 +42,12 @@ class KeywordExtractParam(GenerateParam):
|
||||
- Use comma as a delimiter to separate keywords/phrases.
|
||||
- Answer format: (in language of user's question)
|
||||
- keyword:
|
||||
"""%self.topn
|
||||
""" % self.top_n
|
||||
return self.prompt
|
||||
|
||||
|
||||
class KeywordExtract(Generate, ABC):
|
||||
component_name = "RewriteQuestion"
|
||||
component_name = "KeywordExtract"
|
||||
|
||||
def _run(self, history, **kwargs):
|
||||
q = ""
|
||||
@ -64,5 +63,3 @@ class KeywordExtract(Generate, ABC):
|
||||
ans = re.sub(r".*keyword:", "", ans).strip()
|
||||
if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::")
|
||||
return KeywordExtract.be_output(ans)
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,103 @@
|
||||
{
|
||||
"components": {
|
||||
"begin": {
|
||||
"obj":{
|
||||
"component_name": "Begin",
|
||||
"params": {
|
||||
"prologue": "Hi there!"
|
||||
}
|
||||
},
|
||||
"downstream": ["answer:0"],
|
||||
"upstream": []
|
||||
},
|
||||
"answer:0": {
|
||||
"obj": {
|
||||
"component_name": "Answer",
|
||||
"params": {}
|
||||
},
|
||||
"downstream": ["retrieval:0"],
|
||||
"upstream": ["begin"]
|
||||
},
|
||||
"retrieval:0": {
|
||||
"obj": {
|
||||
"component_name": "Retrieval",
|
||||
"params": {
|
||||
"similarity_threshold": 0.2,
|
||||
"keywords_similarity_weight": 0.3,
|
||||
"top_n": 6,
|
||||
"top_k": 1024,
|
||||
"rerank_id": "BAAI/bge-reranker-v2-m3",
|
||||
"kb_ids": ["21ca4e6a2c8911ef8b1e0242ac120006"],
|
||||
"empty_response": "Sorry, knowledge base has noting related information."
|
||||
}
|
||||
},
|
||||
"downstream": ["relevant:0"],
|
||||
"upstream": ["answer:0"]
|
||||
},
|
||||
"relevant:0": {
|
||||
"obj": {
|
||||
"component_name": "Relevant",
|
||||
"params": {
|
||||
"llm_id": "deepseek-chat",
|
||||
"temperature": 0.02,
|
||||
"yes": "generate:0",
|
||||
"no": "keyword:0"
|
||||
}
|
||||
},
|
||||
"downstream": ["keyword:0", "generate:0"],
|
||||
"upstream": ["retrieval:0"]
|
||||
},
|
||||
"generate:0": {
|
||||
"obj": {
|
||||
"component_name": "Generate",
|
||||
"params": {
|
||||
"llm_id": "deepseek-chat",
|
||||
"prompt": "You are an intelligent assistant. Please answer the question based on content of knowledge base. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\". Answers need to consider chat history.\n Knowledge base content is as following:\n {input}\n The above is the content of knowledge base.",
|
||||
"temperature": 0.2
|
||||
}
|
||||
},
|
||||
"downstream": ["answer:0"],
|
||||
"upstream": ["relevant:0"]
|
||||
},
|
||||
"keyword:0": {
|
||||
"obj": {
|
||||
"component_name": "KeywordExtract",
|
||||
"params": {
|
||||
"llm_id": "deepseek-chat",
|
||||
"prompt": "- Role: You're a question analyzer.\n - Requirements:\n - Summarize user's question, and give top %s important keyword/phrase.\n - Use comma as a delimiter to separate keywords/phrases.\n - Answer format: (in language of user's question)\n - keyword: ",
|
||||
"temperature": 0.2,
|
||||
"top_n": 1
|
||||
}
|
||||
},
|
||||
"downstream": ["baidu:0"],
|
||||
"upstream": ["relevant:0"]
|
||||
},
|
||||
"baidu:0": {
|
||||
"obj":{
|
||||
"component_name": "Baidu",
|
||||
"params": {
|
||||
"top_n": 10
|
||||
}
|
||||
},
|
||||
"downstream": ["generate:1"],
|
||||
"upstream": ["keyword:0"]
|
||||
},
|
||||
"generate:1": {
|
||||
"obj": {
|
||||
"component_name": "Generate",
|
||||
"params": {
|
||||
"llm_id": "deepseek-chat",
|
||||
"prompt": "You are an intelligent assistant. Please answer the question based on content searched from Baidu. When the answer from a Baidu search is incomplete, you need to output the URL link of the corresponding content as well. When all the content searched from Baidu is irrelevant to the question, your answer must include the sentence, \"The answer you are looking for is not found in the Baidu search!\". Answers need to consider chat history.\n The content of Baidu search is as follows:\n {input}\n The above is the content of Baidu search.",
|
||||
"temperature": 0.2
|
||||
}
|
||||
},
|
||||
"downstream": ["answer:0"],
|
||||
"upstream": ["baidu:0"]
|
||||
}
|
||||
},
|
||||
"history": [],
|
||||
"path": [],
|
||||
"messages": [],
|
||||
"reference": {},
|
||||
"answer": []
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user