diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 7cbfda6ae..a7d5c8d53 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1741,6 +1741,11 @@ BING_SEARCH_V7_SUBSCRIPTION_KEY = PersistentConfig( os.environ.get("BING_SEARCH_V7_SUBSCRIPTION_KEY", ""), ) +EXA_API_KEY = PersistentConfig( + "EXA_API_KEY", + "rag.web.search.exa_api_key", + os.getenv("EXA_API_KEY", ""), +) RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig( "RAG_WEB_SEARCH_RESULT_COUNT", diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index a4d63a6d7..2cdebfd4e 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -177,6 +177,7 @@ from open_webui.config import ( BING_SEARCH_V7_ENDPOINT, BING_SEARCH_V7_SUBSCRIPTION_KEY, BRAVE_SEARCH_API_KEY, + EXA_API_KEY, KAGI_SEARCH_API_KEY, MOJEEK_SEARCH_API_KEY, GOOGLE_PSE_API_KEY, @@ -523,6 +524,7 @@ app.state.config.SEARCHAPI_ENGINE = SEARCHAPI_ENGINE app.state.config.JINA_API_KEY = JINA_API_KEY app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY +app.state.config.EXA_API_KEY = EXA_API_KEY app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS diff --git a/backend/open_webui/retrieval/web/exa.py b/backend/open_webui/retrieval/web/exa.py new file mode 100644 index 000000000..3dbcc472e --- /dev/null +++ b/backend/open_webui/retrieval/web/exa.py @@ -0,0 +1,74 @@ +import logging +from dataclasses import dataclass +from typing import Optional + +import requests +from open_webui.env import SRC_LOG_LEVELS +from open_webui.retrieval.web.main import SearchResult + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + +EXA_API_BASE = "https://api.exa.ai" + + +@dataclass +class ExaResult: + url: str + title: str + text: str + + +def search_exa( + api_key: str, + query: str, + count: int, + filter_list: Optional[list[str]] = None, +) -> list[SearchResult]: + """Search using Exa Search API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A Exa Search API key + query (str): The query to search for + count (int): Number of results to return + filter_list (Optional[list[str]]): List of domains to filter results by + """ + log.info(f"Searching with Exa for query: {query}") + + headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + + payload = { + "query": query, + "numResults": count or 5, + "includeDomains": filter_list, + "contents": {"text": True, "highlights": True}, + "type": "auto", # Use the auto search type (keyword or neural) + } + + try: + response = requests.post(f"{EXA_API_BASE}/search", headers=headers, json=payload) + response.raise_for_status() + data = response.json() + + results = [] + for result in data["results"]: + results.append( + ExaResult( + url=result["url"], + title=result["title"], + text=result["text"], + ) + ) + + log.info(f"Found {len(results)} results") + return [ + SearchResult( + link=result.url, + title=result.title, + snippet=result.text, + ) + for result in results + ] + except Exception as e: + log.error(f"Error searching Exa: {e}") + return [] diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 2cffd9ead..35cea6237 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -55,6 +55,7 @@ from open_webui.retrieval.web.serply import search_serply from open_webui.retrieval.web.serpstack import search_serpstack from open_webui.retrieval.web.tavily import search_tavily from open_webui.retrieval.web.bing import search_bing +from open_webui.retrieval.web.exa import search_exa from open_webui.retrieval.utils import ( @@ -388,6 +389,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): "jina_api_key": request.app.state.config.JINA_API_KEY, "bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT, "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY, + "exa_api_key": request.app.state.config.EXA_API_KEY, "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS, }, @@ -436,6 +438,7 @@ class WebSearchConfig(BaseModel): jina_api_key: Optional[str] = None bing_search_v7_endpoint: Optional[str] = None bing_search_v7_subscription_key: Optional[str] = None + exa_api_key: Optional[str] = None result_count: Optional[int] = None concurrent_requests: Optional[int] = None @@ -542,6 +545,8 @@ async def update_rag_config( form_data.web.search.bing_search_v7_subscription_key ) + request.app.state.config.EXA_API_KEY = form_data.web.search.exa_api_key + request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = ( form_data.web.search.result_count ) @@ -591,6 +596,7 @@ async def update_rag_config( "jina_api_key": request.app.state.config.JINA_API_KEY, "bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT, "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY, + "exa_api_key": request.app.state.config.EXA_API_KEY, "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS, }, @@ -1099,6 +1105,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]: - SERPER_API_KEY - SERPLY_API_KEY - TAVILY_API_KEY + - EXA_API_KEY - SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`) Args: query (str): The query to search for @@ -1233,6 +1240,13 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]: request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, ) + elif engine == "exa": + return search_exa( + request.app.state.config.EXA_API_KEY, + query, + request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, + ) else: raise Exception("No search engine API key found in environment variables") diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 58eb09da3..b2accbf1d 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -25,7 +25,8 @@ 'duckduckgo', 'tavily', 'jina', - 'bing' + 'bing', + 'exa' ]; let youtubeLanguage = 'en'; @@ -261,6 +262,17 @@ bind:value={webConfig.search.jina_api_key} /> + {:else if webConfig.search.engine === 'exa'} +