diff --git a/agent/component/crawler.py b/agent/component/crawler.py index f0012d71d..151942b65 100644 --- a/agent/component/crawler.py +++ b/agent/component/crawler.py @@ -17,6 +17,7 @@ from abc import ABC import asyncio from crawl4ai import AsyncWebCrawler from agent.component.base import ComponentBase, ComponentParamBase +from api.utils.web_utils import is_valid_url class CrawlerParam(ComponentParamBase): @@ -39,7 +40,7 @@ class Crawler(ComponentBase, ABC): def _run(self, history, **kwargs): ans = self.get_input() ans = " - ".join(ans["content"]) if "content" in ans else "" - if not ans: + if not is_valid_url(ans): return Crawler.be_output("") try: result = asyncio.run(self.get_web(ans))