Merge remote-tracking branch 'origin/main' into mog/websocket-sdk

.github/archive/publish-rust-sdk.yml

@@ -0,0 +1,42 @@
+name: Publish Rust SDK 
+
+on: []
+
+env:
+  CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          default: true
+          profile: minimal
+
+      - name: Install dependencies
+        run: cargo build --release
+
+      - name: Run version check script
+        id: version_check_script
+        run: |
+          VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name)
+          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+
+      - name: Build the package
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        run: cargo package
+        working-directory: ./apps/rust-sdk
+
+      - name: Publish to crates.io
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        env:
+          CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
+        run: cargo publish
+        working-directory: ./apps/rust-sdk

.github/archive/rust-sdk.yml

@@ -0,0 +1,61 @@
+name: Run Rust SDK E2E Tests
+
+on: []
+
+env:
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  HOST: ${{ secrets.HOST }}
+  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
+  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
+  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
+  PORT: ${{ secrets.PORT }}
+  REDIS_URL: ${{ secrets.REDIS_URL }}
+  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
+  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+  HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
+  HDX_NODE_BETA_MODE: 1
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Install pnpm
+        run: npm install -g pnpm
+      - name: Install dependencies for API
+        run: pnpm install
+        working-directory: ./apps/api
+      - name: Start the application
+        run: npm start &
+        working-directory: ./apps/api
+        id: start_app
+      - name: Start workers
+        run: npm run workers &
+        working-directory: ./apps/api
+        id: start_workers
+      - name: Set up Rust
+        uses: actions/setup-rust@v1
+        with:
+          rust-version: stable
+      - name: Try the lib build
+        working-directory: ./apps/rust-sdk
+        run: cargo build
+      - name: Run E2E tests for Rust SDK
+        run: cargo test --test e2e_with_auth

.github/scripts/check_version_has_incremented.py

@@ -15,6 +15,7 @@ false
 
 """
 import json
+import toml
 import os
 import re
 import sys
@@ -53,6 +54,19 @@ def get_npm_version(package_name: str) -> str:
     version = response.json()['version']
     return version.strip()
 
+def get_rust_version(file_path: str) -> str:
+    """Extract version string from Cargo.toml."""
+    cargo_toml = toml.load(file_path)
+    if 'package' in cargo_toml and 'version' in cargo_toml['package']:
+        return cargo_toml['package']['version'].strip()
+    raise RuntimeError("Unable to find version string in Cargo.toml.")
+
+def get_crates_version(package_name: str) -> str:
+    """Get latest version of Rust package from crates.io."""
+    response = requests.get(f"https://crates.io/api/v1/crates/{package_name}")
+    version = response.json()['crate']['newest_version']
+    return version.strip()
+
 def is_version_incremented(local_version: str, published_version: str) -> bool:
     """Compare local and published versions."""
     local_version_parsed: Version = parse_version(local_version)
@@ -74,6 +88,12 @@ if __name__ == "__main__":
         current_version = get_js_version(os.path.join(package_path, 'package.json'))
         # Get published version from npm
         published_version = get_npm_version(package_name)
+    if package_type == "rust":
+        # Get current version from Cargo.toml
+        current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))
+        # Get published version from crates.io
+        published_version = get_crates_version(package_name)
+
     else:
         raise ValueError("Invalid package type. Use 'python' or 'js'.")
 

.github/scripts/requirements.txt

@@ -1,2 +1,3 @@
 requests
-packaging
+packaging
+toml

.github/workflows/fly.yml

@@ -26,6 +26,7 @@ env:
   PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
   PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
   NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+  CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
   SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
 
 jobs:
@@ -169,7 +170,7 @@ jobs:
       - name: Run E2E tests for JavaScript SDK
         run: npm run test
         working-directory: ./apps/js-sdk/firecrawl
-        
+
   go-sdk-tests:
     name: Go SDK Tests
     needs: pre-deploy-e2e-tests
@@ -205,10 +206,45 @@ jobs:
         run: go test -v ./... -timeout 180s
         working-directory: ./apps/go-sdk/firecrawl
 
+  rust-sdk-tests:
+    name: Rust SDK Tests
+    needs: pre-deploy-e2e-tests
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Install pnpm
+        run: npm install -g pnpm
+      - name: Install dependencies for API
+        run: pnpm install
+        working-directory: ./apps/api
+      - name: Start the application
+        run: npm start &
+        working-directory: ./apps/api
+        id: start_app
+      - name: Start workers
+        run: npm run workers &
+        working-directory: ./apps/api
+        id: start_workers
+      - name: Set up Rust
+        uses: actions/setup-rust@v1
+        with:
+          rust-version: stable
+      - name: Try the lib build
+        working-directory: ./apps/rust-sdk
+        run: cargo build
+      - name: Run E2E tests for Rust SDK
+        run: cargo test --test e2e_with_auth
+
   deploy:
     name: Deploy app
     runs-on: ubuntu-latest
-    needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests]
+    needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests, rust-sdk-tests]
     steps:
       - uses: actions/checkout@v3
       - uses: superfly/flyctl-actions/setup-flyctl@master
@@ -285,7 +321,7 @@ jobs:
       - name: Install dependencies for JavaScript SDK
         run: pnpm install
         working-directory: ./apps/js-sdk/firecrawl
-        
+
       - name: Run version check script
         id: version_check_script
         run: |
@@ -299,4 +335,38 @@ jobs:
         run: |
           npm run build-and-publish
         working-directory: ./apps/js-sdk/firecrawl
-      
+  build-and-publish-rust-sdk:
+    name: Build and publish Rust SDK
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          default: true
+          profile: minimal
+
+      - name: Install dependencies
+        run: cargo build --release
+
+      - name: Run version check script
+        id: version_check_script
+        run: |
+          VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name)
+          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+
+      - name: Build the package
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        run: cargo package
+        working-directory: ./apps/rust-sdk
+
+      - name: Publish to crates.io
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        env:
+          CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
+        run: cargo publish
+        working-directory: ./apps/rust-sdk

apps/js-sdk/firecrawl/src/index.ts

@@ -267,13 +267,10 @@ export interface CrawlStatusResponseV0 {
  * Defines options for mapping URLs during a crawl.
  */
 export interface MapParams {
-  includePaths?: string[]
+  search?: string;
-  excludePaths?: string[]
+  ignoreSitemap?: boolean;
-  maxDepth?: number
+  includeSubdomains?: boolean;
-  limit?: number
+  limit?: number;
-  allowBackwardLinks?: boolean
-  allowExternalLinks?: boolean
-  ignoreSitemap?: boolean
 }
 
 /**

apps/rust-sdk/.gitignore

@@ -0,0 +1 @@
+target/

apps/rust-sdk/CHANGELOG.md

@@ -0,0 +1,7 @@
+## CHANGELOG
+
+## [0.1]
+
+### Added
+
+- [feat] Firecrawl rust sdk.

apps/rust-sdk/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "firecrawl"
+author="Mendable.ai"
+version = "0.1.0"
+edition = "2021"
+license = "GPL-2.0-or-later"
+homepage = "https://www.firecrawl.dev/"
+repository ="https://github.com/mendableai/firecrawl"
+description = "Rust SDK for Firecrawl API."
+authors = ["sanix-darker <sanixdk@gmail.com>"]
+
+[lib]
+path = "src/lib.rs"
+name = "firecrawl"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+reqwest = { version = "^0.12", features = ["json", "blocking"] }
+serde = { version = "^1.0", features = ["derive"] }
+serde_json = "^1.0"
+log = "^0.4"
+thiserror = "^1.0"
+uuid = { version = "^1.10", features = ["v4"] }
+tokio = { version = "^1", features = ["full"] }
+
+[dev-dependencies]
+clippy = "^0.0.302"
+rustfmt = "^0.10"
+assert_matches = "^1.5"
+dotenv = "^0.15"
+tokio = { version = "1", features = ["full"] }
+
+[build-dependencies]
+tokio = { version = "1", features = ["full"] }

apps/rust-sdk/README.md

@@ -0,0 +1,181 @@
+# Firecrawl Rust SDK
+
+The Firecrawl Rust SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
+
+## Installation
+
+To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`:
+
+```toml
+[dependencies]
+firecrawl = "^0.1"
+tokio = { version = "^1", features = ["full"] }
+serde = { version = "^1.0", features = ["derive"] }
+serde_json = "^1.0"
+uuid = { version = "^1.10", features = ["v4"] }
+
+[build-dependencies]
+tokio = { version = "1", features = ["full"] }
+```
+
+To add it in your codebase.
+
+## Usage
+
+1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
+2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` struct.
+
+Here's an example of how to use the SDK in [example.rs](./examples/example.rs):
+All below example can start with :
+```rust
+use firecrawl::FirecrawlApp;
+
+#[tokio::main]
+async fn main() {
+    // Initialize the FirecrawlApp with the API key
+    let api_key = ...;
+    let api_url = ...;
+    let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp");
+
+    // your code here...
+}
+```
+
+### Scraping a URL
+
+To scrape a single URL, use the `scrape_url` method. It takes the URL as a parameter and returns the scraped data as a `serde_json::Value`.
+
+```rust
+// Example scrape code...
+let scrape_result = app.scrape_url("https://example.com", None).await;
+match scrape_result {
+    Ok(data) => println!("Scrape Result:\n{}", data["markdown"]),
+    Err(e) => eprintln!("Scrape failed: {}", e),
+}
+```
+
+### Extracting structured data from a URL
+
+With LLM extraction, you can easily extract structured data from any URL. We support Serde for JSON schema validation to make it easier for you too. Here is how you use it:
+
+```rust
+let json_schema = json!({
+    "type": "object",
+    "properties": {
+        "top": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "title": {"type": "string"},
+                    "points": {"type": "number"},
+                    "by": {"type": "string"},
+                    "commentsURL": {"type": "string"}
+                },
+                "required": ["title", "points", "by", "commentsURL"]
+            },
+            "minItems": 5,
+            "maxItems": 5,
+            "description": "Top 5 stories on Hacker News"
+        }
+    },
+    "required": ["top"]
+});
+
+let llm_extraction_params = json!({
+    "extractorOptions": {
+        "extractionSchema": json_schema,
+        "mode": "llm-extraction"
+    },
+    "pageOptions": {
+        "onlyMainContent": true
+    }
+});
+
+// Example scrape code...
+let llm_extraction_result = app
+    .scrape_url("https://news.ycombinator.com", Some(llm_extraction_params))
+    .await;
+match llm_extraction_result {
+    Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]),
+    Err(e) => eprintln!("LLM Extraction failed: {}", e),
+}
+```
+
+### Search for a query
+
+Used to search the web, get the most relevant results, scrape each page, and return the markdown.
+
+```rust
+// Example query search code...
+let query = "what is mendable?";
+let search_result = app.search(query).await;
+match search_result {
+    Ok(data) => println!("Search Result:\n{}", data),
+    Err(e) => eprintln!("Search failed: {}", e),
+}
+```
+
+### Crawling a Website
+
+To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+
+The `wait_until_done` parameter determines whether the method should wait for the crawl job to complete before returning the result. If set to `true`, the method will periodically check the status of the crawl job until it is completed or the specified `timeout` (in seconds) is reached. If set to `false`, the method will return immediately with the job ID, and you can manually check the status of the crawl job using the `check_crawl_status` method.
+
+```rust
+let random_uuid = String::from(Uuid::new_v4());
+let idempotency_key = Some(random_uuid); // optional idempotency key
+let crawl_params = json!({
+    "crawlerOptions": {
+        "excludes": ["blog/*"]
+    }
+});
+
+// Example crawl code...
+let crawl_result = app
+    .crawl_url("https://example.com", Some(crawl_params), true, 2, idempotency_key)
+    .await;
+match crawl_result {
+    Ok(data) => println!("Crawl Result:\n{}", data),
+    Err(e) => eprintln!("Crawl failed: {}", e),
+}
+```
+
+If `wait_until_done` is set to `true`, the `crawl_url` method will return the crawl result once the job is completed. If the job fails or is stopped, an exception will be raised.
+
+### Checking Crawl Status
+
+To check the status of a crawl job, use the `check_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
+
+```rust
+let job_id = crawl_result["jobId"].as_str().expect("Job ID not found");
+let status = app.check_crawl_status(job_id).await;
+match status {
+    Ok(data) => println!("Crawl Status:\n{}", data),
+    Err(e) => eprintln!("Failed to check crawl status: {}", e),
+}
+```
+
+## Error Handling
+
+The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.
+
+## Running the Tests with Cargo
+
+To ensure the functionality of the Firecrawl Rust SDK, we have included end-to-end tests using `cargo`. These tests cover various aspects of the SDK, including URL scraping, web searching, and website crawling.
+
+### Running the Tests
+
+To run the tests, execute the following commands:
+```bash
+$ export $(xargs < ./tests/.env)
+$ cargo test --test e2e_with_auth
+```
+
+## Contributing
+
+Contributions to the Firecrawl Rust SDK are welcome! If you find any issues or have suggestions for improvements, please open an issue or submit a pull request on the GitHub repository.
+
+## License
+
+The Firecrawl Rust SDK is open-source and released under the [AGPL License](https://www.gnu.org/licenses/agpl-3.0.en.html).

apps/rust-sdk/examples/example.rs

@@ -0,0 +1,82 @@
+use firecrawl::FirecrawlApp;
+use serde_json::json;
+use uuid::Uuid;
+
+#[tokio::main]
+async fn main() {
+    // Initialize the FirecrawlApp with the API key
+    let api_key = Some("fc-YOUR_API_KEY".to_string());
+    let api_url = Some("http://0.0.0.0:3002".to_string());
+    let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp");
+
+    // Scrape a website
+    let scrape_result = app.scrape_url("https://firecrawl.dev", None).await;
+    match scrape_result {
+        Ok(data) => println!("Scrape Result:\n{}", data["markdown"]),
+        Err(e) => eprintln!("Scrape failed: {}", e),
+    }
+
+    // Crawl a website
+    let random_uuid = String::from(Uuid::new_v4());
+    let idempotency_key = Some(random_uuid); // optional idempotency key
+    let crawl_params = json!({
+        "crawlerOptions": {
+            "excludes": ["blog/*"]
+        }
+    });
+    let crawl_result = app
+        .crawl_url(
+            "https://mendable.ai",
+            Some(crawl_params),
+            true,
+            2,
+            idempotency_key,
+        )
+        .await;
+    match crawl_result {
+        Ok(data) => println!("Crawl Result:\n{}", data),
+        Err(e) => eprintln!("Crawl failed: {}", e),
+    }
+
+    // LLM Extraction with a JSON schema
+    let json_schema = json!({
+        "type": "object",
+        "properties": {
+            "top": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "title": {"type": "string"},
+                        "points": {"type": "number"},
+                        "by": {"type": "string"},
+                        "commentsURL": {"type": "string"}
+                    },
+                    "required": ["title", "points", "by", "commentsURL"]
+                },
+                "minItems": 5,
+                "maxItems": 5,
+                "description": "Top 5 stories on Hacker News"
+            }
+        },
+        "required": ["top"]
+    });
+
+    let llm_extraction_params = json!({
+        "extractorOptions": {
+            "extractionSchema": json_schema,
+            "mode": "llm-extraction"
+        },
+        "pageOptions": {
+            "onlyMainContent": true
+        }
+    });
+
+    let llm_extraction_result = app
+        .scrape_url("https://news.ycombinator.com", Some(llm_extraction_params))
+        .await;
+    match llm_extraction_result {
+        Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]),
+        Err(e) => eprintln!("LLM Extraction failed: {}", e),
+    }
+}

apps/rust-sdk/src/lib.rs

@@ -0,0 +1,373 @@
+/*
+*
+* - Structs and Enums:
+*     FirecrawlError: Custom error enum for handling various errors.
+*     FirecrawlApp: Main struct for the application, holding API key, URL, and HTTP client.
+*
+* - Initialization:
+*
+*     FirecrawlApp::new initializes the struct, fetching the API key and URL from environment variables if not provided.
+*
+* - API Methods:
+*     scrape_url, search, crawl_url, check_crawl_status:
+*       Methods for interacting with the Firecrawl API, similar to the Python methods.
+*     monitor_job_status: Polls the API to monitor the status of a crawl job until completion.
+*/
+
+use std::env;
+use std::thread;
+use std::time::Duration;
+
+use log::debug;
+use reqwest::{Client, Response};
+use serde_json::json;
+use serde_json::Value;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum FirecrawlError {
+    #[error("HTTP request failed: {0}")]
+    HttpRequestFailed(String),
+    #[error("API key not provided")]
+    ApiKeyNotProvided,
+    #[error("Failed to parse response: {0}")]
+    ResponseParseError(String),
+    #[error("Crawl job failed or stopped: {0}")]
+    CrawlJobFailed(String),
+}
+
+#[derive(Clone, Debug)]
+pub struct FirecrawlApp {
+    api_key: String,
+    api_url: String,
+    client: Client,
+}
+// the api verstion of firecrawl
+const API_VERSION: &str = "/v0";
+
+impl FirecrawlApp {
+    /// Initialize the FirecrawlApp instance.
+    ///
+    /// # Arguments:
+    ///    * `api_key` (Optional[str]): API key for authenticating with the Firecrawl API.
+    ///    * `api_url` (Optional[str]): Base URL for the Firecrawl API.
+    pub fn new(api_key: Option<String>, api_url: Option<String>) -> Result<Self, FirecrawlError> {
+        let api_key = api_key
+            .or_else(|| env::var("FIRECRAWL_API_KEY").ok())
+            .ok_or(FirecrawlError::ApiKeyNotProvided)?;
+        let api_url = api_url.unwrap_or_else(|| {
+            env::var("FIRECRAWL_API_URL")
+                .unwrap_or_else(|_| "https://api.firecrawl.dev".to_string())
+        });
+
+        debug!("Initialized FirecrawlApp with API key: {}", api_key);
+        debug!("Initialized FirecrawlApp with API URL: {}", api_url);
+
+        Ok(FirecrawlApp {
+            api_key,
+            api_url,
+            client: Client::new(),
+        })
+    }
+
+    /// Scrape the specified URL using the Firecrawl API.
+    ///
+    /// # Arguments:
+    ///    * `url` (str): The URL to scrape.
+    ///    * `params` (Optional[Dict[str, Any]]): Additional parameters for the scrape request.
+    ///
+    /// # Returns:
+    ///    * `Any`: The scraped data if the request is successful.
+    ///
+    /// # Raises:
+    ///    * `Exception`: If the scrape request fails.
+    pub async fn scrape_url(
+        &self,
+        url: &str,
+        params: Option<Value>,
+    ) -> Result<Value, FirecrawlError> {
+        let headers = self.prepare_headers(None);
+        let mut scrape_params = json!({"url": url});
+
+        if let Some(mut params) = params {
+            if let Some(extractor_options) = params.get_mut("extractorOptions") {
+                if let Some(extraction_schema) = extractor_options.get_mut("extractionSchema") {
+                    if extraction_schema.is_object() && extraction_schema.get("schema").is_some() {
+                        extractor_options["extractionSchema"] = extraction_schema["schema"].clone();
+                    }
+                    extractor_options["mode"] = extractor_options
+                        .get("mode")
+                        .cloned()
+                        .unwrap_or_else(|| json!("llm-extraction"));
+                }
+                scrape_params["extractorOptions"] = extractor_options.clone();
+            }
+            for (key, value) in params.as_object().unwrap() {
+                if key != "extractorOptions" {
+                    scrape_params[key] = value.clone();
+                }
+            }
+        }
+
+        let response = self
+            .client
+            .post(&format!("{}{}/scrape", self.api_url, API_VERSION))
+            .headers(headers)
+            .json(&scrape_params)
+            .send()
+            .await
+            .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
+
+        self.handle_response(response, "scrape URL").await
+    }
+
+    /// Perform a search using the Firecrawl API.
+    ///
+    /// # Arguments:
+    ///   * `query` (str): The search query.
+    ///   * `params` (Optional[Dict[str, Any]]): Additional parameters for the search request.
+    ///
+    /// # Returns:
+    ///   * `Any`: The search results if the request is successful.
+    ///
+    /// # Raises:
+    ///   * `Exception`: If the search request fails.
+    pub async fn search(
+        &self,
+        query: &str,
+        params: Option<Value>,
+    ) -> Result<Value, FirecrawlError> {
+        let headers = self.prepare_headers(None);
+        let mut json_data = json!({"query": query});
+        if let Some(params) = params {
+            for (key, value) in params.as_object().unwrap() {
+                json_data[key] = value.clone();
+            }
+        }
+
+        let response = self
+            .client
+            .post(&format!("{}{}/search", self.api_url, API_VERSION))
+            .headers(headers)
+            .json(&json_data)
+            .send()
+            .await
+            .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
+
+        self.handle_response(response, "search").await
+    }
+
+    ///   Initiate a crawl job for the specified URL using the Firecrawl API.
+    ///
+    ///   # Arguments:
+    ///       * `url` (str): The URL to crawl.
+    ///       * `params` (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
+    ///       * `wait_until_done` (bool): Whether to wait until the crawl job is completed.
+    ///       * `poll_interval` (int): Time in seconds between status checks when waiting for job completion.
+    ///       * `idempotency_key` (Optional[str]): A unique uuid key to ensure idempotency of requests.
+    ///
+    ///   # Returns:
+    ///       * `Any`: The crawl job ID or the crawl results if waiting until completion.
+    ///
+    ///   # `Raises`:
+    ///       * `Exception`: If the crawl job initiation or monitoring fails.
+    pub async fn crawl_url(
+        &self,
+        url: &str,
+        params: Option<Value>,
+        wait_until_done: bool,
+        poll_interval: u64,
+        idempotency_key: Option<String>,
+    ) -> Result<Value, FirecrawlError> {
+        let headers = self.prepare_headers(idempotency_key);
+        let mut json_data = json!({"url": url});
+        if let Some(params) = params {
+            for (key, value) in params.as_object().unwrap() {
+                json_data[key] = value.clone();
+            }
+        }
+
+        let response = self
+            .client
+            .post(&format!("{}{}/crawl", self.api_url, API_VERSION))
+            .headers(headers.clone())
+            .json(&json_data)
+            .send()
+            .await
+            .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
+
+        let response_json = self.handle_response(response, "start crawl job").await?;
+        let job_id = response_json["jobId"].as_str().unwrap().to_string();
+
+        if wait_until_done {
+            self.monitor_job_status(&job_id, headers, poll_interval)
+                .await
+        } else {
+            Ok(json!({"jobId": job_id}))
+        }
+    }
+
+    /// Check the status of a crawl job using the Firecrawl API.
+    ///
+    /// # Arguments:
+    ///     * `job_id` (str): The ID of the crawl job.
+    ///
+    /// # Returns:
+    ///     * `Any`: The status of the crawl job.
+    ///
+    /// # Raises:
+    ///     * `Exception`: If the status check request fails.
+    pub async fn check_crawl_status(&self, job_id: &str) -> Result<Value, FirecrawlError> {
+        let headers = self.prepare_headers(None);
+        let response = self
+            .client
+            .get(&format!(
+                "{}{}/crawl/status/{}",
+                self.api_url, API_VERSION, job_id
+            ))
+            .headers(headers)
+            .send()
+            .await
+            .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
+
+        self.handle_response(response, "check crawl status").await
+    }
+
+    /// Monitor the status of a crawl job until completion.
+    ///
+    /// # Arguments:
+    ///     * `job_id` (str): The ID of the crawl job.
+    ///     * `headers` (Dict[str, str]): The headers to include in the status check requests.
+    ///     * `poll_interval` (int): Secounds between status checks.
+    ///
+    /// # Returns:
+    ///     * `Any`: The crawl results if the job is completed successfully.
+    ///
+    /// # Raises:
+    ///     Exception: If the job fails or an error occurs during status checks.
+    async fn monitor_job_status(
+        &self,
+        job_id: &str,
+        headers: reqwest::header::HeaderMap,
+        poll_interval: u64,
+    ) -> Result<Value, FirecrawlError> {
+        loop {
+            let response = self
+                .client
+                .get(&format!(
+                    "{}{}/crawl/status/{}",
+                    self.api_url, API_VERSION, job_id
+                ))
+                .headers(headers.clone())
+                .send()
+                .await
+                .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?;
+
+            let status_data = self.handle_response(response, "check crawl status").await?;
+            match status_data["status"].as_str() {
+                Some("completed") => {
+                    if status_data["data"].is_object() {
+                        return Ok(status_data["data"].clone());
+                    } else {
+                        return Err(FirecrawlError::CrawlJobFailed(
+                            "Crawl job completed but no data was returned".to_string(),
+                        ));
+                    }
+                }
+                Some("active") | Some("paused") | Some("pending") | Some("queued")
+                | Some("waiting") => {
+                    thread::sleep(Duration::from_secs(poll_interval));
+                }
+                Some(status) => {
+                    return Err(FirecrawlError::CrawlJobFailed(format!(
+                        "Crawl job failed or was stopped. Status: {}",
+                        status
+                    )));
+                }
+                None => {
+                    return Err(FirecrawlError::CrawlJobFailed(
+                        "Unexpected response: no status field".to_string(),
+                    ));
+                }
+            }
+        }
+    }
+
+    /// Prepare the headers for API requests.
+    ///
+    /// # Arguments:
+    ///     `idempotency_key` (Optional[str]): A unique key to ensure idempotency of requests.
+    ///
+    /// # Returns:
+    ///     Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.
+    fn prepare_headers(&self, idempotency_key: Option<String>) -> reqwest::header::HeaderMap {
+        let mut headers = reqwest::header::HeaderMap::new();
+        headers.insert("Content-Type", "application/json".parse().unwrap());
+        headers.insert(
+            "Authorization",
+            format!("Bearer {}", self.api_key).parse().unwrap(),
+        );
+        if let Some(key) = idempotency_key {
+            headers.insert("x-idempotency-key", key.parse().unwrap());
+        }
+        headers
+    }
+
+    /// Handle errors from API responses.
+    ///
+    /// # Arguments:
+    ///     * `response` (requests.Response): The response object from the API request.
+    ///     * `action` (str): Description of the action that was being performed.
+    ///
+    /// # Raises:
+    ///     Exception: An exception with a message containing the status code and error details from the response.
+    async fn handle_response(
+        &self,
+        response: Response,
+        action: &str,
+    ) -> Result<Value, FirecrawlError> {
+        if response.status().is_success() {
+            let response_json: Value = response
+                .json()
+                .await
+                .map_err(|e| FirecrawlError::ResponseParseError(e.to_string()))?;
+            if response_json["success"].as_bool().unwrap_or(false) {
+                Ok(response_json["data"].clone())
+            } else {
+                Err(FirecrawlError::HttpRequestFailed(format!(
+                    "Failed to {}: {}",
+                    action, response_json["error"]
+                )))
+            }
+        } else {
+            let status_code = response.status().as_u16();
+            let error_message = response
+                .json::<Value>()
+                .await
+                .unwrap_or_else(|_| json!({"error": "No additional error details provided."}));
+            let message = match status_code {
+                402 => format!(
+                    "Payment Required: Failed to {}. {}",
+                    action, error_message["error"]
+                ),
+                408 => format!(
+                    "Request Timeout: Failed to {} as the request timed out. {}",
+                    action, error_message["error"]
+                ),
+                409 => format!(
+                    "Conflict: Failed to {} due to a conflict. {}",
+                    action, error_message["error"]
+                ),
+                500 => format!(
+                    "Internal Server Error: Failed to {}. {}",
+                    action, error_message["error"]
+                ),
+                _ => format!(
+                    "Unexpected error during {}: Status code {}. {}",
+                    action, status_code, error_message["error"]
+                ),
+            };
+            Err(FirecrawlError::HttpRequestFailed(message))
+        }
+    }
+}

apps/rust-sdk/tests/.env.example

@@ -0,0 +1,2 @@
+API_URL=http://localhost:3002
+TEST_API_KEY=fc-YOUR_API_KEY

apps/rust-sdk/tests/e2e_with_auth.rs

@@ -0,0 +1,174 @@
+use assert_matches::assert_matches;
+use dotenv::dotenv;
+use firecrawl::FirecrawlApp;
+use serde_json::json;
+use std::env;
+use std::time::Duration;
+use tokio::time::sleep;
+
+#[tokio::test]
+async fn test_no_api_key() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").expect("API_URL environment variable is not set");
+    assert_matches!(FirecrawlApp::new(None, Some(api_url)), Err(e) if e.to_string() == "API key not provided");
+}
+
+#[tokio::test]
+async fn test_blocklisted_url() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let blocklisted_url = "https://facebook.com/fake-test";
+    let result = app.scrape_url(blocklisted_url, None).await;
+
+    assert_matches!(
+        result,
+        Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")
+    );
+}
+
+#[tokio::test]
+async fn test_successful_response_with_valid_preview_token() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let app = FirecrawlApp::new(
+        Some("this_is_just_a_preview_token".to_string()),
+        Some(api_url),
+    )
+    .unwrap();
+    let result = app
+        .scrape_url("https://roastmywebsite.ai", None)
+        .await
+        .unwrap();
+    assert!(result.as_object().unwrap().contains_key("content"));
+    assert!(result["content"].as_str().unwrap().contains("_Roast_"));
+}
+
+#[tokio::test]
+async fn test_scrape_url_e2e() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let result = app
+        .scrape_url("https://roastmywebsite.ai", None)
+        .await
+        .unwrap();
+    assert!(result.as_object().unwrap().contains_key("content"));
+    assert!(result.as_object().unwrap().contains_key("markdown"));
+    assert!(result.as_object().unwrap().contains_key("metadata"));
+    assert!(!result.as_object().unwrap().contains_key("html"));
+    assert!(result["content"].as_str().unwrap().contains("_Roast_"));
+}
+
+#[tokio::test]
+async fn test_successful_response_with_valid_api_key_and_include_html() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let params = json!({
+        "pageOptions": {
+            "includeHtml": true
+        }
+    });
+    let result = app
+        .scrape_url("https://roastmywebsite.ai", Some(params))
+        .await
+        .unwrap();
+    assert!(result.as_object().unwrap().contains_key("content"));
+    assert!(result.as_object().unwrap().contains_key("markdown"));
+    assert!(result.as_object().unwrap().contains_key("html"));
+    assert!(result.as_object().unwrap().contains_key("metadata"));
+    assert!(result["content"].as_str().unwrap().contains("_Roast_"));
+    assert!(result["markdown"].as_str().unwrap().contains("_Roast_"));
+    assert!(result["html"].as_str().unwrap().contains("<h1"));
+}
+
+#[tokio::test]
+async fn test_successful_response_for_valid_scrape_with_pdf_file() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let result = app
+        .scrape_url("https://arxiv.org/pdf/astro-ph/9301001.pdf", None)
+        .await
+        .unwrap();
+    assert!(result.as_object().unwrap().contains_key("content"));
+    assert!(result.as_object().unwrap().contains_key("metadata"));
+    assert!(result["content"]
+        .as_str()
+        .unwrap()
+        .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
+}
+
+#[tokio::test]
+async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explicit_extension() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let result = app
+        .scrape_url("https://arxiv.org/pdf/astro-ph/9301001", None)
+        .await
+        .unwrap();
+    sleep(Duration::from_secs(6)).await; // wait for 6 seconds
+    assert!(result.as_object().unwrap().contains_key("content"));
+    assert!(result.as_object().unwrap().contains_key("metadata"));
+    assert!(result["content"]
+        .as_str()
+        .unwrap()
+        .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
+}
+
+#[tokio::test]
+async fn test_should_return_error_for_blocklisted_url() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let blocklisted_url = "https://twitter.com/fake-test";
+    let result = app.crawl_url(blocklisted_url, None, true, 1, None).await;
+
+    assert_matches!(
+        result,
+        Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.")
+    );
+}
+
+#[tokio::test]
+async fn test_llm_extraction() {
+    dotenv().ok();
+    let api_url = env::var("API_URL").unwrap();
+    let api_key = env::var("TEST_API_KEY").unwrap();
+    let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap();
+    let params = json!({
+        "extractorOptions": {
+            "mode": "llm-extraction",
+            "extractionPrompt": "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+            "extractionSchema": {
+                "type": "object",
+                "properties": {
+                    "company_mission": {"type": "string"},
+                    "supports_sso": {"type": "boolean"},
+                    "is_open_source": {"type": "boolean"}
+                },
+                "required": ["company_mission", "supports_sso", "is_open_source"]
+            }
+        }
+    });
+    let result = app
+        .scrape_url("https://mendable.ai", Some(params))
+        .await
+        .unwrap();
+    assert!(result.as_object().unwrap().contains_key("llm_extraction"));
+    let llm_extraction = &result["llm_extraction"];
+    assert!(llm_extraction
+        .as_object()
+        .unwrap()
+        .contains_key("company_mission"));
+    assert!(llm_extraction["supports_sso"].is_boolean());
+    assert!(llm_extraction["is_open_source"].is_boolean());
+}