diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts index d10a286f..af094442 100644 --- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts @@ -449,4 +449,64 @@ describe("E2E Tests for v1 API Routes", () => { }); + +describe("POST /v1/map", () => { + it.concurrent("should require authorization", async () => { + const response: ScrapeResponseRequestTest = await request(TEST_URL).post( + "/v1/map" + ); + expect(response.statusCode).toBe(401); + }); + + it.concurrent("should return an error response with an invalid API key", async () => { + const response: ScrapeResponseRequestTest = await request(TEST_URL) + .post("/v1/map") + .set("Authorization", `Bearer invalid-api-key`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + expect(response.statusCode).toBe(401); + }); + + it.concurrent("should return a successful response with a valid API key", async () => { + const mapRequest = { + url: "https://roastmywebsite.ai", + includeSubdomains: true, + search: "test", + }; + + const response: ScrapeResponseRequestTest = await request(TEST_URL) + .post("/v1/map") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send(mapRequest); + + expect(response.statusCode).toBe(200); + expect(response.body).toHaveProperty("success", true); + expect(response.body).toHaveProperty("links"); + if (!("links" in response.body)) { + throw new Error("Expected response body to have 'links' property"); + } + const links = response.body.links as unknown[]; + expect(Array.isArray(links)).toBe(true); + expect(links.length).toBeGreaterThan(0); + }); + + it.concurrent("should return an error for invalid URL", async () => { + const mapRequest = { + url: "invalid-url", + includeSubdomains: true, + search: "test", + }; + + const response: ScrapeResponseRequestTest = await request(TEST_URL) + .post("/v1/map") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send(mapRequest); + + expect(response.statusCode).toBe(400); + expect(response.body).toHaveProperty("success", false); + expect(response.body).toHaveProperty("error"); + }); +}); }); diff --git a/apps/api/src/controllers/v1/auth.ts b/apps/api/src/controllers/v1/auth.ts index bc6951c9..d4da3c6b 100644 --- a/apps/api/src/controllers/v1/auth.ts +++ b/apps/api/src/controllers/v1/auth.ts @@ -115,6 +115,9 @@ export async function supaAuthenticateUser( case RateLimiterMode.CrawlStatus: rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); break; + case RateLimiterMode.Map: + rateLimiter = getRateLimiter(RateLimiterMode.Map, token); + break; case RateLimiterMode.Preview: rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); @@ -151,7 +154,7 @@ export async function supaAuthenticateUser( if ( token === "this_is_just_a_preview_token" && - (mode === RateLimiterMode.Scrape || mode === RateLimiterMode.Preview || mode === RateLimiterMode.Search) + (mode === RateLimiterMode.Scrape || mode === RateLimiterMode.Preview || mode === RateLimiterMode.Search || mode === RateLimiterMode.Map) ) { return { success: true, team_id: "preview" }; // check the origin of the request and make sure its from firecrawl.dev diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts index 0807bc0f..25e12c63 100644 --- a/apps/api/src/routes/v1.ts +++ b/apps/api/src/routes/v1.ts @@ -71,7 +71,7 @@ function idempotencyMiddleware(req: Request, res: Response, next: NextFunction) } function blocklistMiddleware(req: Request, res: Response, next: NextFunction) { - if (isUrlBlocked(req.body.url)) { + if (req.body.url && isUrlBlocked(req.body.url)) { return res.status(403).json({ success: false, error: "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." }); } next(); @@ -108,7 +108,7 @@ v1Router.post( v1Router.post( "/map", blocklistMiddleware, - authMiddleware(RateLimiterMode.Crawl), + authMiddleware(RateLimiterMode.Map), checkCreditsMiddleware(1), wrap(mapController) ); diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index f1399b13..2682d0a2 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -42,6 +42,19 @@ const RATE_LIMITS = { growth: 500, growthdouble: 500, }, + map:{ + default: 20, + free: 5, + starter: 20, + standard: 40, + standardOld: 40, + scale: 500, + hobby: 10, + standardNew: 50, + standardnew: 50, + growth: 500, + growthdouble: 500, + }, preview: { free: 5, default: 5, diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 5e63ac78..70a8ab07 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -106,6 +106,7 @@ export enum RateLimiterMode { Scrape = "scrape", Preview = "preview", Search = "search", + Map = "map", }