AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 02:55:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #373 from Sanix-Darker/f/rust-sdk

Browse Source 
[Feat]: Add RUST SDK client for firecrawl API
This commit is contained in:
Rafael Miller 2024-08-29 16:23:13 -03:00 committed by GitHub
commit 1ecee90305
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 3052 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
.github/archive/publish-rust-sdk.yml vendored Normal file
View File

@ -0,0 +1,42 @@
name: Publish Rust SDK
on: []
env:
CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
jobs:
build-and-publish:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
default: true
profile: minimal
- name: Install dependencies
run: cargo build --release
- name: Run version check script
id: version_check_script
run: |
VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name)
echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
- name: Build the package
if: ${{ env.VERSION_INCREMENTED == 'true' }}
run: cargo package
working-directory: ./apps/rust-sdk
- name: Publish to crates.io
if: ${{ env.VERSION_INCREMENTED == 'true' }}
env:
CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
run: cargo publish
working-directory: ./apps/rust-sdk

61
.github/archive/rust-sdk.yml vendored Normal file
View File

@ -0,0 +1,61 @@
name: Run Rust SDK E2E Tests
on: []
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
HOST: ${{ secrets.HOST }}
LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
PORT: ${{ secrets.PORT }}
REDIS_URL: ${{ secrets.REDIS_URL }}
SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
HDX_NODE_BETA_MODE: 1
jobs:
build:
runs-on: ubuntu-latest
services:
redis:
image: redis
ports:
- 6379:6379
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Install pnpm
run: npm install -g pnpm
- name: Install dependencies for API
run: pnpm install
working-directory: ./apps/api
- name: Start the application
run: npm start &
working-directory: ./apps/api
id: start_app
- name: Start workers
run: npm run workers &
working-directory: ./apps/api
id: start_workers
- name: Set up Rust
uses: actions/setup-rust@v1
with:
rust-version: stable
- name: Try the lib build
working-directory: ./apps/rust-sdk
run: cargo build
- name: Run E2E tests for Rust SDK
run: cargo test --test e2e_with_auth

20
.github/scripts/check_version_has_incremented.py vendored
View File

@ -15,6 +15,7 @@ false
"""
import json
import toml
import os
import re
import sys
@ -53,6 +54,19 @@ def get_npm_version(package_name: str) -> str:
version = response.json()['version']
return version.strip()
def get_rust_version(file_path: str) -> str:
"""Extract version string from Cargo.toml."""
cargo_toml = toml.load(file_path)
if 'package' in cargo_toml and 'version' in cargo_toml['package']:
return cargo_toml['package']['version'].strip()
raise RuntimeError("Unable to find version string in Cargo.toml.")
def get_crates_version(package_name: str) -> str:
"""Get latest version of Rust package from crates.io."""
response = requests.get(f"https://crates.io/api/v1/crates/{package_name}")
version = response.json()['crate']['newest_version']
return version.strip()
def is_version_incremented(local_version: str, published_version: str) -> bool:
"""Compare local and published versions."""
local_version_parsed: Version = parse_version(local_version)
@ -74,6 +88,12 @@ if __name__ == "__main__":
current_version = get_js_version(os.path.join(package_path, 'package.json'))
# Get published version from npm
published_version = get_npm_version(package_name)
if package_type == "rust":
# Get current version from Cargo.toml
current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))
# Get published version from crates.io
published_version = get_crates_version(package_name)
else:
raise ValueError("Invalid package type. Use 'python' or 'js'.")

1
.github/scripts/requirements.txt vendored
View File

@ -1,2 +1,3 @@
requests
packaging
toml

72
.github/workflows/fly.yml vendored
View File

@ -26,6 +26,7 @@ env:
PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
jobs:
@ -205,10 +206,45 @@ jobs:
run: go test -v ./... -timeout 180s
working-directory: ./apps/go-sdk/firecrawl
rust-sdk-tests:
name: Rust SDK Tests
needs: pre-deploy-e2e-tests
runs-on: ubuntu-latest
services:
redis:
image: redis
ports:
- 6379:6379
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Install pnpm
run: npm install -g pnpm
- name: Install dependencies for API
run: pnpm install
working-directory: ./apps/api
- name: Start the application
run: npm start &
working-directory: ./apps/api
id: start_app
- name: Start workers
run: npm run workers &
working-directory: ./apps/api
id: start_workers
- name: Set up Rust
uses: actions/setup-rust@v1
with:
rust-version: stable
- name: Try the lib build
working-directory: ./apps/rust-sdk
run: cargo build
- name: Run E2E tests for Rust SDK
run: cargo test --test e2e_with_auth
deploy:
name: Deploy app
runs-on: ubuntu-latest
needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests]
needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests, rust-sdk-tests]
steps:
- uses: actions/checkout@v3
- uses: superfly/flyctl-actions/setup-flyctl@master
@ -299,4 +335,38 @@ jobs:
run: |
npm run build-and-publish
working-directory: ./apps/js-sdk/firecrawl
build-and-publish-rust-sdk:
name: Build and publish Rust SDK
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
default: true
profile: minimal
- name: Install dependencies
run: cargo build --release
- name: Run version check script
id: version_check_script
run: |
VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name)
echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
- name: Build the package
if: ${{ env.VERSION_INCREMENTED == 'true' }}
run: cargo package
working-directory: ./apps/rust-sdk
- name: Publish to crates.io
if: ${{ env.VERSION_INCREMENTED == 'true' }}
env:
CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
run: cargo publish
working-directory: ./apps/rust-sdk

1
apps/rust-sdk/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
target/

7
apps/rust-sdk/CHANGELOG.md Normal file
View File

@ -0,0 +1,7 @@
## CHANGELOG
## [0.1]
### Added
- [feat] Firecrawl rust sdk.

1999
apps/rust-sdk/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

34
apps/rust-sdk/Cargo.toml Normal file
View File

@ -0,0 +1,34 @@
[package]
name = "firecrawl"
author="Mendable.ai"
version = "0.1.0"
edition = "2021"
license = "GPL-2.0-or-later"
homepage = "https://www.firecrawl.dev/"
repository ="https://github.com/mendableai/firecrawl"
description = "Rust SDK for Firecrawl API."
authors = ["sanix-darker <sanixdk@gmail.com>"]
[lib]
path = "src/lib.rs"
name = "firecrawl"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
reqwest = { version = "^0.12", features = ["json", "blocking"] }
serde = { version = "^1.0", features = ["derive"] }
serde_json = "^1.0"
log = "^0.4"
thiserror = "^1.0"
uuid = { version = "^1.10", features = ["v4"] }
tokio = { version = "^1", features = ["full"] }
[dev-dependencies]
clippy = "^0.0.302"
rustfmt = "^0.10"
assert_matches = "^1.5"
dotenv = "^0.15"
tokio = { version = "1", features = ["full"] }
[build-dependencies]
tokio = { version = "1", features = ["full"] }

181
apps/rust-sdk/README.md Normal file
View File

@ -0,0 +1,181 @@
# Firecrawl Rust SDK
The Firecrawl Rust SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
## Installation
To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`:
```toml
[dependencies]
firecrawl = "^0.1"
tokio = { version = "^1", features = ["full"] }
serde = { version = "^1.0", features = ["derive"] }
serde_json = "^1.0"
uuid = { version = "^1.10", features = ["v4"] }
[build-dependencies]
tokio = { version = "1", features = ["full"] }
```
To add it in your codebase.
## Usage
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` struct.
Here's an example of how to use the SDK in [example.rs](./examples/example.rs):
All below example can start with :
```rust
use firecrawl::FirecrawlApp;
#[tokio::main]
async fn main() {
// Initialize the FirecrawlApp with the API key
let api_key = ...;
let api_url = ...;
let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp");
// your code here...
}
```
### Scraping a URL
To scrape a single URL, use the `scrape_url` method. It takes the URL as a parameter and returns the scraped data as a `serde_json::Value`.
```rust
// Example scrape code...
let scrape_result = app.scrape_url("https://example.com", None).await;
match scrape_result {
Ok(data) => println!("Scrape Result:\n{}", data["markdown"]),
Err(e) => eprintln!("Scrape failed: {}", e),
}
```
### Extracting structured data from a URL
With LLM extraction, you can easily extract structured data from any URL. We support Serde for JSON schema validation to make it easier for you too. Here is how you use it:
```rust
let json_schema = json!({
"type": "object",
"properties": {
"top": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {"type": "string"},
"points": {"type": "number"},
"by": {"type": "string"},
"commentsURL": {"type": "string"}
},
"required": ["title", "points", "by", "commentsURL"]
},
"minItems": 5,
"maxItems": 5,
"description": "Top 5 stories on Hacker News"
}
},
"required": ["top"]
});
let llm_extraction_params = json!({
"extractorOptions": {
"extractionSchema": json_schema,
"mode": "llm-extraction"
},
"pageOptions": {
"onlyMainContent": true
}
});
// Example scrape code...
let llm_extraction_result = app
.scrape_url("https://news.ycombinator.com", Some(llm_extraction_params))
.await;
match llm_extraction_result {
Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]),
Err(e) => eprintln!("LLM Extraction failed: {}", e),
}
```
### Search for a query
Used to search the web, get the most relevant results, scrape each page, and return the markdown.
```rust
// Example query search code...
let query = "what is mendable?";
let search_result = app.search(query).await;
match search_result {
Ok(data) => println!("Search Result:\n{}", data),
Err(e) => eprintln!("Search failed: {}", e),
}
```
### Crawling a Website
To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
The `wait_until_done` parameter determines whether the method should wait for the crawl job to complete before returning the result. If set to `true`, the method will periodically check the status of the crawl job until it is completed or the specified `timeout` (in seconds) is reached. If set to `false`, the method will return immediately with the job ID, and you can manually check the status of the crawl job using the `check_crawl_status` method.
```rust
let random_uuid = String::from(Uuid::new_v4());
let idempotency_key = Some(random_uuid); // optional idempotency key
let crawl_params = json!({
"crawlerOptions": {
"excludes": ["blog/*"]
}
});
// Example crawl code...
let crawl_result = app
.crawl_url("https://example.com", Some(crawl_params), true, 2, idempotency_key)
.await;
match crawl_result {
Ok(data) => println!("Crawl Result:\n{}", data),
Err(e) => eprintln!("Crawl failed: {}", e),
}
```
If `wait_until_done` is set to `true`, the `crawl_url` method will return the crawl result once the job is completed. If the job fails or is stopped, an exception will be raised.
### Checking Crawl Status
To check the status of a crawl job, use the `check_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
```rust
let job_id = crawl_result["jobId"].as_str().expect("Job ID not found");
let status = app.check_crawl_status(job_id).await;
match status {
Ok(data) => println!("Crawl Status:\n{}", data),
Err(e) => eprintln!("Failed to check crawl status: {}", e),
}
```
## Error Handling
The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.
## Running the Tests with Cargo
To ensure the functionality of the Firecrawl Rust SDK, we have included end-to-end tests using `cargo`. These tests cover various aspects of the SDK, including URL scraping, web searching, and website crawling.
### Running the Tests
To run the tests, execute the following commands:
```bash
$ export $(xargs < ./tests/.env)
$ cargo test --test e2e_with_auth
```
## Contributing
Contributions to the Firecrawl Rust SDK are welcome! If you find any issues or have suggestions for improvements, please open an issue or submit a pull request on the GitHub repository.
## License
The Firecrawl Rust SDK is open-source and released under the [AGPL License](https://www.gnu.org/licenses/agpl-3.0.en.html).

82
apps/rust-sdk/examples/example.rs Normal file
View File

@ -0,0 +1,82 @@
use firecrawl::FirecrawlApp;
use serde_json::json;
use uuid::Uuid;
#[tokio::main]
async fn main() {
// Initialize the FirecrawlApp with the API key
let api_key = Some("fc-YOUR_API_KEY".to_string());
let api_url = Some("http://0.0.0.0:3002".to_string());
let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp");
// Scrape a website
let scrape_result = app.scrape_url("https://firecrawl.dev", None).await;
match scrape_result {
Ok(data) => println!("Scrape Result:\n{}", data["markdown"]),
Err(e) => eprintln!("Scrape failed: {}", e),
}
// Crawl a website
let random_uuid = String::from(Uuid::new_v4());
let idempotency_key = Some(random_uuid); // optional idempotency key
let crawl_params = json!({
"crawlerOptions": {
"excludes": ["blog/*"]
}
});
let crawl_result = app
.crawl_url(
"https://mendable.ai",
Some(crawl_params),
true,
2,
idempotency_key,
)
.await;
match crawl_result {
Ok(data) => println!("Crawl Result:\n{}", data),
Err(e) => eprintln!("Crawl failed: {}", e),
}
// LLM Extraction with a JSON schema
let json_schema = json!({
"type": "object",
"properties": {
"top": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {"type": "string"},
"points": {"type": "number"},
"by": {"type": "string"},
"commentsURL": {"type": "string"}
},
"required": ["title", "points", "by", "commentsURL"]
},
"minItems": 5,
"maxItems": 5,
"description": "Top 5 stories on Hacker News"
}
},
"required": ["top"]
});
let llm_extraction_params = json!({
"extractorOptions": {
"extractionSchema": json_schema,
"mode": "llm-extraction"
},
"pageOptions": {
"onlyMainContent": true
}
});
let llm_extraction_result = app
.scrape_url("https://news.ycombinator.com", Some(llm_extraction_params))
.await;
match llm_extraction_result {
Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]),
Err(e) => eprintln!("LLM Extraction failed: {}", e),
}
}

373
apps/rust-sdk/src/lib.rs Normal file
View File

@ -0,0 +1,373 @@
/*
*
* - Structs and Enums:
* FirecrawlError: Custom error enum for handling various errors.
* FirecrawlApp: Main struct for the application, holding API key, URL, and HTTP client.
*
* - Initialization:
*
* FirecrawlApp::new initializes the struct, fetching the API key and URL from environment variables if not provided.
*
* - API Methods:
* scrape_url, search, crawl_url, check_crawl_status:
* Methods for interacting with the Firecrawl API, similar to the Python methods.
* monitor_job_status: Polls the API to monitor the status of a crawl job until completion.
*/
use std::env;
use std::thread;
use std::time::Duration;
use log::debug;
use reqwest::{Client, Response};
use serde_json::json;
use serde_json::Value;
use thiserror::Error;
#[derive(Error, Debug)]
pub enum FirecrawlError {
#[error("HTTP request failed: {0}")]
HttpRequestFailed(String),
#[error("API key not provided")]
ApiKeyNotProvided,
#[error("Failed to parse response: {0}")]
ResponseParseError(String),
#[error("Crawl job failed or stopped: {0}")]
CrawlJobFailed(String),
}
#[derive(Clone, Debug)]
pub struct FirecrawlApp {
api_key: String,
api_url: String,
client: Client,
}
// the api verstion of firecrawl
const API_VERSION: &str = "/v0";
impl FirecrawlApp {
/// Initialize the FirecrawlApp instance.
///
/// # Arguments:
/// * `api_key` (Optional[str]): API key for authenticating with the Firecrawl API.
/// * `api_url` (Optional[str]): Base URL for the Firecrawl API.
pub fn new(api_key: Option<String>, api_url: Option<String>) -> Result<Self, FirecrawlError> {
let api_key = api_key
.or_else(|| env::var("FIRECRAWL_API_KEY").ok())
.ok_or(FirecrawlError::ApiKeyNotProvided)?;
let api_url = api_url.unwrap_or_else(|| {
env::var("FIRECRAWL_API_URL")
.unwrap_or_else(|_| "https://api.firecrawl.dev".to_string())
});
debug!("Initialized FirecrawlApp with API key: {}", api_key);
debug!("Initialized FirecrawlApp with API URL: {}", api_url);
Ok(FirecrawlApp {
api_key,
api_url,
client: Client::new(),
})
}
/// Scrape the specified URL using the Firecrawl API.
///
/// # Arguments:
/// * `url` (str): The URL to scrape.
/// * `params` (Optional[Dict[str, Any]]): Additional parameters for the scrape request.
///
/// # Returns:
/// * `Any`: The scraped data if the request is successful.
///
/// # Raises:
/// * `Exception`: If the scrape request fails.
pub async fn scrape_url(
&self,
url: &str,
params: Option<Value>,
) -> Result<Value, FirecrawlError> {
let headers = self.prepare_headers(None);
let mut scrape_params = json!({"url": url});
if let Some(mut params) = params {
if let Some(extractor_options) = params.get_mut("extractorOptions") {
if let Some(extraction_schema) = extractor_options.get_mut("extractionSchema") {
if extraction_schema.is_object() && extraction_schema.get("schema").is_some() {
extractor_options["extractionSchema"] = extraction_schema["schema"].clone();
}
extractor_options["mode"] = extractor_options
.get("mode")
.cloned()
.unwrap_or_else(|| json!("llm-extraction"));
}
scrape_params["extractorOptions"] = extractor_options.clone();
}
for (key, value) in params.as_object().unwrap() {
if key != "extractorOptions" {
scrape_params[key] = value.clone();
}
}
}
let response = self
.client
.post(&format!("{}{}/scrape", self.api_url, API_VERSION))
.headers(headers)
.json(&scrape_params)
.send()
.await
.map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
self.handle_response(response, "scrape URL").await
}
/// Perform a search using the Firecrawl API.
///
/// # Arguments:
/// * `query` (str): The search query.
/// * `params` (Optional[Dict[str, Any]]): Additional parameters for the search request.
///
/// # Returns:
/// * `Any`: The search results if the request is successful.
///
/// # Raises:
/// * `Exception`: If the search request fails.
pub async fn search(
&self,
query: &str,
params: Option<Value>,
) -> Result<Value, FirecrawlError> {
let headers = self.prepare_headers(None);
let mut json_data = json!({"query": query});
if let Some(params) = params {
for (key, value) in params.as_object().unwrap() {
json_data[key] = value.clone();
}
}
let response = self
.client
.post(&format!("{}{}/search", self.api_url, API_VERSION))
.headers(headers)
.json(&json_data)
.send()
.await
.map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
self.handle_response(response, "search").await
}
/// Initiate a crawl job for the specified URL using the Firecrawl API.
///
/// # Arguments:
/// * `url` (str): The URL to crawl.
/// * `params` (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
/// * `wait_until_done` (bool): Whether to wait until the crawl job is completed.
/// * `poll_interval` (int): Time in seconds between status checks when waiting for job completion.
/// * `idempotency_key` (Optional[str]): A unique uuid key to ensure idempotency of requests.
///
/// # Returns:
/// * `Any`: The crawl job ID or the crawl results if waiting until completion.
///
/// # `Raises`:
/// * `Exception`: If the crawl job initiation or monitoring fails.
pub async fn crawl_url(
&self,
url: &str,
params: Option<Value>,
wait_until_done: bool,
poll_interval: u64,
idempotency_key: Option<String>,
) -> Result<Value, FirecrawlError> {
let headers = self.prepare_headers(idempotency_key);
let mut json_data = json!({"url": url});
if let Some(params) = params {
for (key, value) in params.as_object().unwrap() {
json_data[key] = value.clone();
}
}
let response = self
.client
.post(&format!("{}{}/crawl", self.api_url, API_VERSION))
.headers(headers.clone())
.json(&json_data)
.send()
.await
.map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
let response_json = self.handle_response(response, "start crawl job").await?;
let job_id = response_json["jobId"].as_str().unwrap().to_string();
if wait_until_done {
self.monitor_job_status(&job_id, headers, poll_interval)
.await
} else {
Ok(json!({"jobId": job_id}))
}
}
/// Check the status of a crawl job using the Firecrawl API.
///
/// # Arguments:
/// * `job_id` (str): The ID of the crawl job.
///
/// # Returns:
/// * `Any`: The status of the crawl job.
///
/// # Raises:
/// * `Exception`: If the status check request fails.
pub async fn check_crawl_status(&self, job_id: &str) -> Result<Value, FirecrawlError> {
let headers = self.prepare_headers(None);
let response = self
.client
.get(&format!(
"{}{}/crawl/status/{}",
self.api_url, API_VERSION, job_id
))
.headers(headers)
.send()
.await
.map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
self.handle_response(response, "check crawl status").await
}
/// Monitor the status of a crawl job until completion.
///
/// # Arguments:
/// * `job_id` (str): The ID of the crawl job.
/// * `headers` (Dict[str, str]): The headers to include in the status check requests.
/// * `poll_interval` (int): Secounds between status checks.
///
/// # Returns:
/// * `Any`: The crawl results if the job is completed successfully.
///
/// # Raises:
/// Exception: If the job fails or an error occurs during status checks.
async fn monitor_job_status(
&self,
job_id: &str,
headers: reqwest::header::HeaderMap,
poll_interval: u64,
) -> Result<Value, FirecrawlError> {
loop {
let response = self
.client
.get(&format!(
"{}{}/crawl/status/{}",
self.api_url, API_VERSION, job_id
))
.headers(headers.clone())
.send()
.await
.map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
let status_data = self.handle_response(response, "check crawl status").await?;
match status_data["status"].as_str() {
Some("completed") => {
if status_data["data"].is_object() {
return Ok(status_data["data"].clone());
} else {
return Err(FirecrawlError::CrawlJobFailed(
"Crawl job completed but no data was returned".to_string(),
));
}
}
Some("active") | Some("paused") | Some("pending") | Some("queued")
| Some("waiting") => {
thread::sleep(Duration::from_secs(poll_interval));
}
Some(status) => {
return Err(FirecrawlError::CrawlJobFailed(format!(
"Crawl job failed or was stopped. Status: {}",
status
)));
}
None => {
return Err(FirecrawlError::CrawlJobFailed(
"Unexpected response: no status field".to_string(),
));
}
}
}
}
/// Prepare the headers for API requests.
///
/// # Arguments:
/// `idempotency_key` (Optional[str]): A unique key to ensure idempotency of requests.
///
/// # Returns:
/// Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.
fn prepare_headers(&self, idempotency_key: Option<String>) -> reqwest::header::HeaderMap {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert("Content-Type", "application/json".parse().unwrap());
headers.insert(
"Authorization",
format!("Bearer {}", self.api_key).parse().unwrap(),
);
if let Some(key) = idempotency_key {
headers.insert("x-idempotency-key", key.parse().unwrap());
}
headers
}
/// Handle errors from API responses.
///
/// # Arguments:
/// * `response` (requests.Response): The response object from the API request.
/// * `action` (str): Description of the action that was being performed.
///
/// # Raises:
/// Exception: An exception with a message containing the status code and error details from the response.
async fn handle_response(
&self,
response: Response,
action: &str,
) -> Result<Value, FirecrawlError> {
if response.status().is_success() {
let response_json: Value = response
.json()
.await
.map_err(|e| FirecrawlError::ResponseParseError(e.to_string()))?;
if response_json["success"].as_bool().unwrap_or(false) {
Ok(response_json["data"].clone())
} else {
Err(FirecrawlError::HttpRequestFailed(format!(
"Failed to {}: {}",
action, response_json["error"]
)))
}
} else {
let status_code = response.status().as_u16();
let error_message = response
.json::<Value>()
.await
.unwrap_or_else(|_| json!({"error": "No additional error details provided."}));
let message = match status_code {
402 => format!(
"Payment Required: Failed to {}. {}",
action, error_message["error"]
),
408 => format!(
"Request Timeout: Failed to {} as the request timed out. {}",
action, error_message["error"]
),
409 => format!(
"Conflict: Failed to {} due to a conflict. {}",
action, error_message["error"]
),
500 => format!(
"Internal Server Error: Failed to {}. {}",
action, error_message["error"]
),
_ => format!(
"Unexpected error during {}: Status code {}. {}",
action, status_code, error_message["error"]
),
};
Err(FirecrawlError::HttpRequestFailed(message))
}
}
}

2
apps/rust-sdk/tests/.env.example Normal file
View File

@ -0,0 +1,2 @@
API_URL=http://localhost:3002
TEST_API_KEY=fc-YOUR_API_KEY

174
apps/rust-sdk/tests/e2e_with_auth.rs Normal file
View File

@ -0,0 +1,174 @@
use assert_matches::assert_matches;
use dotenv::dotenv;
use firecrawl::FirecrawlApp;
use serde_json::json;
use std::env;
use std::time::Duration;
use tokio::time::sleep;
#[tokio::test]
async fn test_no_api_key() {
dotenv().ok();
let api_url = env::var("API_URL").expect("API_URL environment variable is not set");
assert_matches!(FirecrawlApp::new(None, Some(api_url)), Err(e) if e.to_string() == "API key not provided");
}
#[tokio::test]
async fn test_blocklisted_url() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let blocklisted_url = "https://facebook.com/fake-test";
let result = app.scrape_url(blocklisted_url, None).await;
assert_matches!(
result,
Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")
);
}
#[tokio::test]
async fn test_successful_response_with_valid_preview_token() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let app = FirecrawlApp::new(
Some("this_is_just_a_preview_token".to_string()),
Some(api_url),
)
.unwrap();
let result = app
.scrape_url("https://roastmywebsite.ai", None)
.await
.unwrap();
assert!(result.as_object().unwrap().contains_key("content"));
assert!(result["content"].as_str().unwrap().contains("_Roast_"));
}
#[tokio::test]
async fn test_scrape_url_e2e() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let result = app
.scrape_url("https://roastmywebsite.ai", None)
.await
.unwrap();
assert!(result.as_object().unwrap().contains_key("content"));
assert!(result.as_object().unwrap().contains_key("markdown"));
assert!(result.as_object().unwrap().contains_key("metadata"));
assert!(!result.as_object().unwrap().contains_key("html"));
assert!(result["content"].as_str().unwrap().contains("_Roast_"));
}
#[tokio::test]
async fn test_successful_response_with_valid_api_key_and_include_html() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let params = json!({
"pageOptions": {
"includeHtml": true
}
});
let result = app
.scrape_url("https://roastmywebsite.ai", Some(params))
.await
.unwrap();
assert!(result.as_object().unwrap().contains_key("content"));
assert!(result.as_object().unwrap().contains_key("markdown"));
assert!(result.as_object().unwrap().contains_key("html"));
assert!(result.as_object().unwrap().contains_key("metadata"));
assert!(result["content"].as_str().unwrap().contains("_Roast_"));
assert!(result["markdown"].as_str().unwrap().contains("_Roast_"));
assert!(result["html"].as_str().unwrap().contains("<h1"));
}
#[tokio::test]
async fn test_successful_response_for_valid_scrape_with_pdf_file() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let result = app
.scrape_url("https://arxiv.org/pdf/astro-ph/9301001.pdf", None)
.await
.unwrap();
assert!(result.as_object().unwrap().contains_key("content"));
assert!(result.as_object().unwrap().contains_key("metadata"));
assert!(result["content"]
.as_str()
.unwrap()
.contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
}
#[tokio::test]
async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let result = app
.scrape_url("https://arxiv.org/pdf/astro-ph/9301001", None)
.await
.unwrap();
sleep(Duration::from_secs(6)).await; // wait for 6 seconds
assert!(result.as_object().unwrap().contains_key("content"));
assert!(result.as_object().unwrap().contains_key("metadata"));
assert!(result["content"]
.as_str()
.unwrap()
.contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
}
#[tokio::test]
async fn test_should_return_error_for_blocklisted_url() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let blocklisted_url = "https://twitter.com/fake-test";
let result = app.crawl_url(blocklisted_url, None, true, 1, None).await;
assert_matches!(
result,
Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.")
);
}
#[tokio::test]
async fn test_llm_extraction() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").unwrap();
let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
let params = json!({
"extractorOptions": {
"mode": "llm-extraction",
"extractionPrompt": "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
"extractionSchema": {
"type": "object",
"properties": {
"company_mission": {"type": "string"},
"supports_sso": {"type": "boolean"},
"is_open_source": {"type": "boolean"}
},
"required": ["company_mission", "supports_sso", "is_open_source"]
}
}
});
let result = app
.scrape_url("https://mendable.ai", Some(params))
.await
.unwrap();
assert!(result.as_object().unwrap().contains_key("llm_extraction"));
let llm_extraction = &result["llm_extraction"];
assert!(llm_extraction
.as_object()
.unwrap()
.contains_key("company_mission"));
assert!(llm_extraction["supports_sso"].is_boolean());
assert!(llm_extraction["is_open_source"].is_boolean());
}