From 55a4a3d7681f83d7b23f043767175f89f569ddcf Mon Sep 17 00:00:00 2001 From: rafaelmmiller <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 6 Mar 2025 18:16:07 -0300 Subject: [PATCH 1/3] preview-token --- apps/api/src/controllers/auth.ts | 6 +++--- apps/api/src/controllers/v0/crawlPreview.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index a5aed8ff..bfcc5ac0 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -200,7 +200,7 @@ export async function supaAuthenticateUser( let priceId: string | null = null; let chunk: AuthCreditUsageChunk | null = null; let plan: PlanType = "free"; - if (token == "this_is_just_a_preview_token") { + if (token == process.env.PREVIEW_TOKEN) { if (mode == RateLimiterMode.CrawlStatus) { rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); } else if (mode == RateLimiterMode.ExtractStatus) { @@ -295,7 +295,7 @@ export async function supaAuthenticateUser( } const team_endpoint_token = - token === "this_is_just_a_preview_token" ? iptoken : teamId; + token === process.env.PREVIEW_TOKEN ? iptoken : teamId; try { await rateLimiter.consume(team_endpoint_token); @@ -325,7 +325,7 @@ export async function supaAuthenticateUser( } if ( - token === "this_is_just_a_preview_token" && + token === process.env.PREVIEW_TOKEN && (mode === RateLimiterMode.Scrape || mode === RateLimiterMode.Preview || mode === RateLimiterMode.Map || diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts index 00776e53..ffb8ebba 100644 --- a/apps/api/src/controllers/v0/crawlPreview.ts +++ b/apps/api/src/controllers/v0/crawlPreview.ts @@ -24,7 +24,7 @@ export async function crawlPreviewController(req: Request, res: Response) { const incomingIP = (req.headers["x-forwarded-for"] || req.socket.remoteAddress) as string; - const iptoken = incomingIP + "this_is_just_a_preview_token"; + const iptoken = incomingIP + process.env.PREVIEW_TOKEN; const team_id = `preview_${iptoken}`; if (!auth.success) { From b66e0ccb7db4e044888300508d6df9cec1b85c54 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 6 Mar 2025 18:36:01 -0300 Subject: [PATCH 2/3] Nick: --- .../__tests__/e2e_full_withAuth/index.test.ts | 26 ++----------------- .../src/__tests__/e2e_withAuth/index.test.ts | 2 +- .../__tests__/v1/e2e_withAuth/index.test.ts | 4 +-- .../firecrawl/__tests__/e2e_withAuth/test.py | 2 +- .../__tests__/v1/e2e_withAuth/test.py | 4 +-- apps/rust-sdk/tests/e2e_with_auth.rs | 2 +- 6 files changed, 9 insertions(+), 31 deletions(-) diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts index 40686c45..45b29d2b 100644 --- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts @@ -62,15 +62,6 @@ describe("E2E Tests for API Routes", () => { expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE); }); - // tested on rate limit test - // it.concurrent("should return a successful response with a valid preview token", async () => { - // const response = await request(TEST_URL) - // .post("/v0/scrape") - // .set("Authorization", `Bearer this_is_just_a_preview_token`) - // .set("Content-Type", "application/json") - // .send({ url: "https://roastmywebsite.ai" }); - // expect(response.statusCode).toBe(200); - // }, 30000); // 30 seconds timeout it.concurrent( "should return a successful response with a valid API key", @@ -1087,19 +1078,6 @@ describe("E2E Tests for API Routes", () => { }, 3000, ); - - // it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => { - // const response = await request(TEST_URL) - // .post("/v0/crawlWebsitePreview") - // .set("Authorization", `Bearer this_is_just_a_preview_token`) - // .set("Content-Type", "application/json") - // .send({ url: "https://firecrawl.dev" }); - // expect(response.statusCode).toBe(200); - // expect(response.body).toHaveProperty("jobId"); - // expect(response.body.jobId).toMatch( - // /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/ - // ); - // }); }); describe("POST /v0/search", () => { @@ -1724,7 +1702,7 @@ describe("E2E Tests for API Routes", () => { for (let i = 0; i < 5; i++) { const response = await request(TEST_URL) .post("/v0/scrape") - .set("Authorization", `Bearer this_is_just_a_preview_token`) + .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`) .set("Content-Type", "application/json") .send({ url: "https://www.scrapethissite.com" }); @@ -1732,7 +1710,7 @@ describe("E2E Tests for API Routes", () => { } const response = await request(TEST_URL) .post("/v0/scrape") - .set("Authorization", `Bearer this_is_just_a_preview_token`) + .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`) .set("Content-Type", "application/json") .send({ url: "https://www.scrapethissite.com" }); diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts index 7d107afe..02bd7847 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts @@ -55,7 +55,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => { "should return successful response with valid preview token", async () => { const app = new FirecrawlApp<"v0">({ - apiKey: "this_is_just_a_preview_token", + apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL, version: "v0", }); diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts index 2e601dc4..d689b014 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts @@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => { }); test.concurrent('should return successful response with valid preview token', async () => { - const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL }); + const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL }); const response = await app.scrapeUrl('https://roastmywebsite.ai'); if (!response.success) { throw new Error(response.error); @@ -365,7 +365,7 @@ describe('FirecrawlApp E2E Tests', () => { }); test.concurrent('should return successful response with valid preview token', async () => { - const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL }); + const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL }); const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse; expect(response).not.toBeNull(); expect(response.links?.length).toBeGreaterThan(0); diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py index 50d5306f..bbe5df68 100644 --- a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py +++ b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py @@ -37,7 +37,7 @@ def test_scrape_url_invalid_api_key(): # assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value) def test_successful_response_with_valid_preview_token(): - app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0') + app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'), version='v0') response = app.scrape_url('https://roastmywebsite.ai') assert response is not None assert 'content' in response diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py index eacec8da..d99ee820 100644 --- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py +++ b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py @@ -49,7 +49,7 @@ def test_scrape_url_invalid_api_key(): # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) def test_successful_response_with_valid_preview_token(): - app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token") + app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN')) response = app.scrape_url('https://roastmywebsite.ai') assert response is not None assert "_Roast_" in response['markdown'] @@ -327,7 +327,7 @@ def test_invalid_api_key_on_map(): # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) def test_successful_response_with_valid_preview_token_on_map(): - app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL) + app = FirecrawlApp(api_key=os.getenv('PREVIEW_TOKEN'), api_url=API_URL) response = app.map_url('https://roastmywebsite.ai') assert response is not None assert len(response) > 0 diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs index 00f3e26c..882a2941 100644 --- a/apps/rust-sdk/tests/e2e_with_auth.rs +++ b/apps/rust-sdk/tests/e2e_with_auth.rs @@ -26,7 +26,7 @@ async fn test_successful_response_with_valid_preview_token() { let api_url = env::var("API_URL").unwrap(); let app = FirecrawlApp::new_selfhosted( api_url, - Some("this_is_just_a_preview_token"), + Some(env::var("PREVIEW_TOKEN").unwrap()), ) .unwrap(); let result = app From 1ced546bd08f5118d9754d69e9633cef38dba235 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 6 Mar 2025 19:17:02 -0300 Subject: [PATCH 3/3] Update rate-limiter.ts --- apps/api/src/services/rate-limiter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 480c18a4..4b4af827 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -137,8 +137,8 @@ const RATE_LIMITS = { extract_pro: 1000, }, preview: { - free: 0, - default: 0, + free: 5, + default: 5, }, account: { free: 100,