From 7f891939f183e75a1a38ff09935dd1d1cdb0a8ea Mon Sep 17 00:00:00 2001 From: Yash Parmar <82636823+Yash-1511@users.noreply.github.com> Date: Tue, 5 Mar 2024 07:53:44 +0530 Subject: [PATCH] FEAT: add tavily tool for searching... A search engine for LLM (#2681) --- .../provider/builtin/tavily/_assets/icon.png | Bin 0 -> 2070 bytes .../tools/provider/builtin/tavily/tavily.py | 22 +++ .../tools/provider/builtin/tavily/tavily.yaml | 29 ++++ .../builtin/tavily/tools/tavily_search.py | 161 ++++++++++++++++++ .../builtin/tavily/tools/tavily_search.yaml | 27 +++ 5 files changed, 239 insertions(+) create mode 100644 api/core/tools/provider/builtin/tavily/_assets/icon.png create mode 100644 api/core/tools/provider/builtin/tavily/tavily.py create mode 100644 api/core/tools/provider/builtin/tavily/tavily.yaml create mode 100644 api/core/tools/provider/builtin/tavily/tools/tavily_search.py create mode 100644 api/core/tools/provider/builtin/tavily/tools/tavily_search.yaml diff --git a/api/core/tools/provider/builtin/tavily/_assets/icon.png b/api/core/tools/provider/builtin/tavily/_assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..fdb40ab5689ba9f40b22d2c700ed2ce1b2602829 GIT binary patch literal 2070 zcmV+x2(^!000yK02}}S4*(A$00|%Q@$&cg z_|?|f=;`XMud<(^r8GH2ZE$zW%gtL}W{QlFgocd5!^XP1zI=a$-{9hMb$gVSnn+Al zKSND1HbPEPS{EKJAtyDasH~Wpo?&Ec8zL|~KugHU%PB8AlScC&000L;Nkl zhnA`^5Qa^F01*LuLq&JjwO-%Do~`A?FmAv78_H8nLgH8nLgH8nK? zGV4KGz?$_8+5*<#KC}gl@$nAY1jh976xsyFEXWet1jY=N3A71p=_t+++6K1xnEpcB zz?QyksBLH?nBpl2%bnD$9i)$;jbMsqAWiC|riT$yJZLLeG9w7NduS_IvQiO}J6ud$ zgb?mPo57F+4HR~v&0t61Xpq#-G;lO$XS#s4gB?>fP-se=dmqg}pN$oYb{gcV;-s1RXjOBm6EEKWJOnFuyC5wY(U)H-+X8v@L8n zSHCx~4{Zz!#`;|#vIfxBu)tIP6v~;nT`H&hHkGj9HpUMErQD{=)}ulhx2bA8DwNQh z{)PK%(3h?Zp198n4Hep=lIdaef-PCM=2kdK>uWj+eN~9CRF!_;eiMjXsm9$W^If4- z0b~5;XD$I_1Z`J&+}jr~izrQU8oh)vE#i0d63TNDBwOJg#tq?_*Xkb3+xZ8lJrMxKq)D>e?)`yU{<0K;x)|o~|bX zEKFGRJ`*U8L@!4me3dA~M*VP$h7bYV&4{GyNI>I0XsoqD0?%>r$aQb3{gb8SO~(E8 zp=HF=ZRO2mqA>X_`nNc}E0YBSb9xbc^m&wcg?w>XYIHaGQ_*Vvc;b` zpjq73&=_sdJBH zeg57wTK1x%dvNW+2`g3W3osPCW?xrz<`dm2Hv`RsOIn`KRNh`AEb_A)&?f%bvDvM? z{vHz-Vt2k`T6nlCIV&gr`m}eG4aVnF=TLsUN@h&g!U3%$iQU*T%n# zd5QU3uRlUK!)?(@pbBG85j*fkH+IoF7rDdS#vXTHWw zpoMgZx~Qbp?|YV3eXeS__^Z0nX=l69tJP?z`pBpJclgO=(rj13|uH!>XiNi-_%-)(O@H;f|bdJr!0&UK3dan~77|4Lm{fE!{Vz(4U_^4!A zL?W;EpPwlpQ4E0vB#I)i(HMP}i0cBJ!5foLJgUvT7k%Wrm9FRo(LR$sU`(O!z| z<-D3n6^>K2XyV^6i^34`)vs+HiDLeLb)Wq+A&bj`H=W2gJ)>}hdr;wupRwpS#C-&_ zpY14${4Cu%OU>`Dq0KBnTT73Mh9+Djd?dp&JRpPq zSI*o$v`Hn?d41tUmnPcZ7(M$M`U`DP>2#lcjck4=6Tf-!&ePB3BuYvDc-p2i z{)hDWXR_i>Dw0_JZBz3IZ4470wyCv)HiiwxeVfYYNdxXdpUPAjJ~F?xzZHQlKjor* zJBUQ0_Qf-|v<7qi7;+I7ip;Sh;QH%oRl=i_@% literal 0 HcmV?d00001 diff --git a/api/core/tools/provider/builtin/tavily/tavily.py b/api/core/tools/provider/builtin/tavily/tavily.py new file mode 100644 index 0000000000..a013d41fcf --- /dev/null +++ b/api/core/tools/provider/builtin/tavily/tavily.py @@ -0,0 +1,22 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.tavily.tools.tavily_search import TavilySearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class TavilyProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + TavilySearchTool().fork_tool_runtime( + meta={ + "credentials": credentials, + } + ).invoke( + user_id='', + tool_parameters={ + "query": "Sachin Tendulkar", + }, + ) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/tavily/tavily.yaml b/api/core/tools/provider/builtin/tavily/tavily.yaml new file mode 100644 index 0000000000..50826e37b3 --- /dev/null +++ b/api/core/tools/provider/builtin/tavily/tavily.yaml @@ -0,0 +1,29 @@ +identity: + author: Yash Parmar + name: tavily + label: + en_US: Tavily + zh_Hans: Tavily + pt_BR: Tavily + description: + en_US: Tavily + zh_Hans: Tavily + pt_BR: Tavily + icon: icon.png +credentials_for_provider: + tavily_api_key: + type: secret-input + required: true + label: + en_US: Tavily API key + zh_Hans: Tavily API key + pt_BR: Tavily API key + placeholder: + en_US: Please input your Tavily API key + zh_Hans: 请输入你的 Tavily API key + pt_BR: Please input your Tavily API key + help: + en_US: Get your Tavily API key from Tavily + zh_Hans: 从 TavilyApi 获取您的 Tavily API key + pt_BR: Get your Tavily API key from Tavily + url: https://docs.tavily.com/docs/tavily-api/introduction diff --git a/api/core/tools/provider/builtin/tavily/tools/tavily_search.py b/api/core/tools/provider/builtin/tavily/tools/tavily_search.py new file mode 100644 index 0000000000..9a4d27376b --- /dev/null +++ b/api/core/tools/provider/builtin/tavily/tools/tavily_search.py @@ -0,0 +1,161 @@ +from typing import Any, Optional + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + +TAVILY_API_URL = "https://api.tavily.com" + + +class TavilySearch: + """ + A class for performing search operations using the Tavily Search API. + + Args: + api_key (str): The API key for accessing the Tavily Search API. + + Methods: + raw_results: Retrieves raw search results from the Tavily Search API. + results: Retrieves cleaned search results from the Tavily Search API. + clean_results: Cleans the raw search results. + """ + + def __init__(self, api_key: str) -> None: + self.api_key = api_key + + def raw_results( + self, + query: str, + max_results: Optional[int] = 3, + search_depth: Optional[str] = "advanced", + include_domains: Optional[list[str]] = [], + exclude_domains: Optional[list[str]] = [], + include_answer: Optional[bool] = False, + include_raw_content: Optional[bool] = False, + include_images: Optional[bool] = False, + ) -> dict: + """ + Retrieves raw search results from the Tavily Search API. + + Args: + query (str): The search query. + max_results (int, optional): The maximum number of results to retrieve. Defaults to 3. + search_depth (str, optional): The search depth. Defaults to "advanced". + include_domains (List[str], optional): The domains to include in the search. Defaults to []. + exclude_domains (List[str], optional): The domains to exclude from the search. Defaults to []. + include_answer (bool, optional): Whether to include answer in the search results. Defaults to False. + include_raw_content (bool, optional): Whether to include raw content in the search results. Defaults to False. + include_images (bool, optional): Whether to include images in the search results. Defaults to False. + + Returns: + dict: The raw search results. + + """ + params = { + "api_key": self.api_key, + "query": query, + "max_results": max_results, + "search_depth": search_depth, + "include_domains": include_domains, + "exclude_domains": exclude_domains, + "include_answer": include_answer, + "include_raw_content": include_raw_content, + "include_images": include_images, + } + response = requests.post(f"{TAVILY_API_URL}/search", json=params) + response.raise_for_status() + return response.json() + + def results( + self, + query: str, + max_results: Optional[int] = 3, + search_depth: Optional[str] = "advanced", + include_domains: Optional[list[str]] = [], + exclude_domains: Optional[list[str]] = [], + include_answer: Optional[bool] = False, + include_raw_content: Optional[bool] = False, + include_images: Optional[bool] = False, + ) -> list[dict]: + """ + Retrieves cleaned search results from the Tavily Search API. + + Args: + query (str): The search query. + max_results (int, optional): The maximum number of results to retrieve. Defaults to 3. + search_depth (str, optional): The search depth. Defaults to "advanced". + include_domains (List[str], optional): The domains to include in the search. Defaults to []. + exclude_domains (List[str], optional): The domains to exclude from the search. Defaults to []. + include_answer (bool, optional): Whether to include answer in the search results. Defaults to False. + include_raw_content (bool, optional): Whether to include raw content in the search results. Defaults to False. + include_images (bool, optional): Whether to include images in the search results. Defaults to False. + + Returns: + list: The cleaned search results. + + """ + raw_search_results = self.raw_results( + query, + max_results=max_results, + search_depth=search_depth, + include_domains=include_domains, + exclude_domains=exclude_domains, + include_answer=include_answer, + include_raw_content=include_raw_content, + include_images=include_images, + ) + return self.clean_results(raw_search_results["results"]) + + def clean_results(self, results: list[dict]) -> list[dict]: + """ + Cleans the raw search results. + + Args: + results (list): The raw search results. + + Returns: + list: The cleaned search results. + + """ + clean_results = [] + for result in results: + clean_results.append( + { + "url": result["url"], + "content": result["content"], + } + ) + # return clean results as a string + return "\n".join([f"{res['url']}\n{res['content']}" for res in clean_results]) + + +class TavilySearchTool(BuiltinTool): + """ + A tool for searching Tavily using a given query. + """ + + def _invoke( + self, user_id: str, tool_parameters: dict[str, Any] + ) -> ToolInvokeMessage | list[ToolInvokeMessage]: + """ + Invokes the Tavily search tool with the given user ID and tool parameters. + + Args: + user_id (str): The ID of the user invoking the tool. + tool_parameters (Dict[str, Any]): The parameters for the Tavily search tool. + + Returns: + ToolInvokeMessage | list[ToolInvokeMessage]: The result of the Tavily search tool invocation. + """ + query = tool_parameters.get("query", "") + api_key = self.runtime.credentials["tavily_api_key"] + if not query: + return self.create_text_message("Please input query") + tavily_search = TavilySearch(api_key) + results = tavily_search.results(query) + print(results) + if not results: + return self.create_text_message(f"No results found for '{query}' in Tavily") + else: + return self.create_text_message(text=results) diff --git a/api/core/tools/provider/builtin/tavily/tools/tavily_search.yaml b/api/core/tools/provider/builtin/tavily/tools/tavily_search.yaml new file mode 100644 index 0000000000..ccdb9408fc --- /dev/null +++ b/api/core/tools/provider/builtin/tavily/tools/tavily_search.yaml @@ -0,0 +1,27 @@ +identity: + name: tavily_search + author: Yash Parmar + label: + en_US: TavilySearch + zh_Hans: TavilySearch + pt_BR: TavilySearch +description: + human: + en_US: A tool for search engine built specifically for AI agents (LLMs), delivering real-time, accurate, and factual results at speed. + zh_Hans: 专为人工智能代理 (LLM) 构建的搜索引擎工具，可快速提供实时、准确和真实的结果。 + pt_BR: A tool for search engine built specifically for AI agents (LLMs), delivering real-time, accurate, and factual results at speed. + llm: A tool for search engine built specifically for AI agents (LLMs), delivering real-time, accurate, and factual results at speed. +parameters: + - name: query + type: string + required: true + label: + en_US: Query string + zh_Hans: 查询语句 + pt_BR: Query string + human_description: + en_US: used for searching + zh_Hans: 用于搜索网页内容 + pt_BR: used for searching + llm_description: key words for searching + form: llm