AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-14 06:25:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix/firecrawl parameters issue (#6213)

Browse Source
This commit is contained in:
Matri 2024-07-12 12:59:50 +08:00 committed by GitHub
parent 7b225a5ab0
commit a9ee52f2d7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/core/rag/extractor/firecrawl/firecrawl_app.py
View File

@ -46,7 +46,6 @@ class FirecrawlApp:
raise Exception(f'Failed to scrape URL. Status code: {response.status_code}') raise Exception(f'Failed to scrape URL. Status code: {response.status_code}')
def crawl_url(self, url, params=None) -> str: def crawl_url(self, url, params=None) -> str:
start_time = time.time()
headers = self._prepare_headers() headers = self._prepare_headers()
json_data = {'url': url} json_data = {'url': url}
if params: if params:

7
api/core/tools/provider/builtin/firecrawl/firecrawl_appx.py
View File

@ -1,3 +1,4 @@
import logging
import time import time
from collections.abc import Mapping from collections.abc import Mapping
from typing import Any from typing import Any
@ -5,6 +6,7 @@ from typing import Any
import requests import requests
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
logger = logging.getLogger(__name__)
class FirecrawlApp: class FirecrawlApp:
def __init__(self, api_key: str | None = None, base_url: str | None = None): def __init__(self, api_key: str | None = None, base_url: str | None = None):
@ -48,6 +50,7 @@ class FirecrawlApp:
headers = self._prepare_headers() headers = self._prepare_headers()
data = {'url': url, **kwargs} data = {'url': url, **kwargs}
response = self._request('POST', endpoint, data, headers) response = self._request('POST', endpoint, data, headers)
logger.debug(f"Sent request to {endpoint=} body={data}")
if response is None: if response is None:
raise HTTPError("Failed to scrape URL after multiple retries") raise HTTPError("Failed to scrape URL after multiple retries")
return response return response
@ -57,6 +60,7 @@ class FirecrawlApp:
headers = self._prepare_headers() headers = self._prepare_headers()
data = {'query': query, **kwargs} data = {'query': query, **kwargs}
response = self._request('POST', endpoint, data, headers) response = self._request('POST', endpoint, data, headers)
logger.debug(f"Sent request to {endpoint=} body={data}")
if response is None: if response is None:
raise HTTPError("Failed to perform search after multiple retries") raise HTTPError("Failed to perform search after multiple retries")
return response return response
@ -66,8 +70,9 @@ class FirecrawlApp:
): ):
endpoint = f'{self.base_url}/v0/crawl' endpoint = f'{self.base_url}/v0/crawl'
headers = self._prepare_headers(idempotency_key) headers = self._prepare_headers(idempotency_key)
data = {'url': url, **kwargs} data = {'url': url, **kwargs['params']}
response = self._request('POST', endpoint, data, headers) response = self._request('POST', endpoint, data, headers)
logger.debug(f"Sent request to {endpoint=} body={data}")
if response is None: if response is None:
raise HTTPError("Failed to initiate crawl after multiple retries") raise HTTPError("Failed to initiate crawl after multiple retries")
job_id: str = response['jobId'] job_id: str = response['jobId']