From bdc2b74e8fc06651882dc131a2999f5bd60d16b1 Mon Sep 17 00:00:00 2001 From: Hayden Zhou Date: Fri, 23 May 2025 09:48:55 +0800 Subject: [PATCH] Fix baidu request error (#7799) ### What problem does this PR solve? ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) Co-authored-by: xiaohzho --- agent/component/baidu.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/agent/component/baidu.py b/agent/component/baidu.py index daec9f058..b75faa4a0 100644 --- a/agent/component/baidu.py +++ b/agent/component/baidu.py @@ -17,6 +17,7 @@ import logging from abc import ABC import pandas as pd import requests +from bs4 import BeautifulSoup import re from agent.component.base import ComponentBase, ComponentParamBase @@ -44,17 +45,28 @@ class Baidu(ComponentBase, ABC): return Baidu.be_output("") try: - url = 'http://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n) + url = 'https://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n) headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'} + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + 'Connection': 'keep-alive', + } response = requests.get(url=url, headers=headers) - - url_res = re.findall(r"'url': \\\"(.*?)\\\"}", response.text) - title_res = re.findall(r"'title': \\\"(.*?)\\\",\\n", response.text) - body_res = re.findall(r"\"contentText\":\"(.*?)\"", response.text) - baidu_res = [{"content": re.sub('|', '', '' + title + ' ' + body)} for - url, title, body in zip(url_res, title_res, body_res)] - del body_res, url_res, title_res + # check if request success + if response.status_code == 200: + soup = BeautifulSoup(response.text, 'html.parser') + url_res = [] + title_res = [] + body_res = [] + for item in soup.select('.result.c-container'): + # extract title + title_res.append(item.select_one('h3 a').get_text(strip=True)) + url_res.append(item.select_one('h3 a')['href']) + body_res.append(item.select_one('.c-abstract').get_text(strip=True) if item.select_one('.c-abstract') else '') + baidu_res = [{"content": re.sub('|', '', '' + title + ' ' + body)} for + url, title, body in zip(url_res, title_res, body_res)] + del body_res, url_res, title_res except Exception as e: return Baidu.be_output("**ERROR**: " + str(e))