AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-30 00:45:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: temp file fixes

Browse Source
This commit is contained in:
Nicolas 2025-01-13 23:56:53 -03:00
parent ac6650e488
commit 9759f18725
3 changed files with 80 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
View File

@ -2,7 +2,6 @@ import request from "supertest";
import { configDotenv } from "dotenv"; import { configDotenv } from "dotenv";
import { import {
ScrapeRequestInput, ScrapeRequestInput,
ScrapeResponseRequestTest,
} from "../../controllers/v1/types"; } from "../../controllers/v1/types";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
@ -20,7 +19,7 @@ describe("E2E Tests for v1 API Routes", () => {
describe("GET /is-production", () => { describe("GET /is-production", () => {
it.concurrent("should return the production status", async () => { it.concurrent("should return the production status", async () => {
const response: ScrapeResponseRequestTest = const response: any =
await request(TEST_URL).get("/is-production"); await request(TEST_URL).get("/is-production");
console.log( console.log(
@ -39,7 +38,7 @@ describe("E2E Tests for v1 API Routes", () => {
describe("POST /v1/scrape", () => { describe("POST /v1/scrape", () => {
it.concurrent("should require authorization", async () => { it.concurrent("should require authorization", async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.send({ url: "https://firecrawl.dev" }); .send({ url: "https://firecrawl.dev" });
@ -64,7 +63,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return an error response with an invalid API key", "should return an error response with an invalid API key",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -80,7 +79,7 @@ describe("E2E Tests for v1 API Routes", () => {
url: "https://roastmywebsite.ai", url: "https://roastmywebsite.ai",
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -132,7 +131,7 @@ describe("E2E Tests for v1 API Routes", () => {
url: "https://arxiv.org/abs/2410.04840", url: "https://arxiv.org/abs/2410.04840",
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -189,7 +188,7 @@ describe("E2E Tests for v1 API Routes", () => {
formats: ["markdown", "html"], formats: ["markdown", "html"],
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -217,7 +216,7 @@ describe("E2E Tests for v1 API Routes", () => {
url: "https://arxiv.org/pdf/astro-ph/9301001.pdf", url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",
// formats: ["markdown", "html"], // formats: ["markdown", "html"],
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -245,7 +244,7 @@ describe("E2E Tests for v1 API Routes", () => {
const scrapeRequest: ScrapeRequestInput = { const scrapeRequest: ScrapeRequestInput = {
url: "https://arxiv.org/pdf/astro-ph/9301001", url: "https://arxiv.org/pdf/astro-ph/9301001",
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -275,7 +274,7 @@ describe("E2E Tests for v1 API Routes", () => {
url: "https://www.scrapethissite.com/", url: "https://www.scrapethissite.com/",
onlyMainContent: false, // default is true onlyMainContent: false, // default is true
}; };
const responseWithoutRemoveTags: ScrapeResponseRequestTest = const responseWithoutRemoveTags: any =
await request(TEST_URL) await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@ -302,7 +301,7 @@ describe("E2E Tests for v1 API Routes", () => {
excludeTags: [".nav", "#footer", "strong"], excludeTags: [".nav", "#footer", "strong"],
onlyMainContent: false, // default is true onlyMainContent: false, // default is true
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -325,7 +324,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 400 page", "should return a successful response for a scrape with 400 page",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -347,7 +346,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 401 page", "should return a successful response for a scrape with 401 page",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -368,7 +367,7 @@ describe("E2E Tests for v1 API Routes", () => {
// Removed it as we want to retry fallback to the next scraper // Removed it as we want to retry fallback to the next scraper
// it.concurrent('should return a successful response for a scrape with 403 page', async () => { // it.concurrent('should return a successful response for a scrape with 403 page', async () => {
// const response: ScrapeResponseRequestTest = await request(TEST_URL) // const response: any = await request(TEST_URL)
// .post('/v1/scrape') // .post('/v1/scrape')
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
// .set('Content-Type', 'application/json') // .set('Content-Type', 'application/json')
@ -388,7 +387,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 404 page", "should return a successful response for a scrape with 404 page",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -408,7 +407,7 @@ describe("E2E Tests for v1 API Routes", () => {
); );
// it.concurrent('should return a successful response for a scrape with 405 page', async () => { // it.concurrent('should return a successful response for a scrape with 405 page', async () => {
// const response: ScrapeResponseRequestTest = await request(TEST_URL) // const response: any = await request(TEST_URL)
// .post('/v1/scrape') // .post('/v1/scrape')
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
// .set('Content-Type', 'application/json') // .set('Content-Type', 'application/json')
@ -426,7 +425,7 @@ describe("E2E Tests for v1 API Routes", () => {
// }, 60000); // }, 60000);
// it.concurrent('should return a successful response for a scrape with 500 page', async () => { // it.concurrent('should return a successful response for a scrape with 500 page', async () => {
// const response: ScrapeResponseRequestTest = await request(TEST_URL) // const response: any = await request(TEST_URL)
// .post('/v1/scrape') // .post('/v1/scrape')
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
// .set('Content-Type', 'application/json') // .set('Content-Type', 'application/json')
@ -446,7 +445,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return a timeout error when scraping takes longer than the specified timeout", "should return a timeout error when scraping takes longer than the specified timeout",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -465,7 +464,7 @@ describe("E2E Tests for v1 API Routes", () => {
formats: ["html", "rawHtml"], formats: ["html", "rawHtml"],
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -497,7 +496,7 @@ describe("E2E Tests for v1 API Routes", () => {
waitFor: 8000, waitFor: 8000,
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -528,7 +527,7 @@ describe("E2E Tests for v1 API Routes", () => {
formats: ["links"], formats: ["links"],
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -553,7 +552,7 @@ describe("E2E Tests for v1 API Routes", () => {
describe("POST /v1/map", () => { describe("POST /v1/map", () => {
it.concurrent("should require authorization", async () => { it.concurrent("should require authorization", async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.send({ url: "https://firecrawl.dev" }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
@ -562,7 +561,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return an error response with an invalid API key", "should return an error response with an invalid API key",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -578,7 +577,7 @@ describe("E2E Tests for v1 API Routes", () => {
url: "https://roastmywebsite.ai", url: "https://roastmywebsite.ai",
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -604,7 +603,7 @@ describe("E2E Tests for v1 API Routes", () => {
search: "pricing", search: "pricing",
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -632,7 +631,7 @@ describe("E2E Tests for v1 API Routes", () => {
includeSubdomains: true, includeSubdomains: true,
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -664,7 +663,7 @@ describe("E2E Tests for v1 API Routes", () => {
includeSubdomains: true, includeSubdomains: true,
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -697,7 +696,7 @@ describe("E2E Tests for v1 API Routes", () => {
includeSubdomains: false, includeSubdomains: false,
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -723,7 +722,7 @@ describe("E2E Tests for v1 API Routes", () => {
search: "test", search: "test",
}; };
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/map") .post("/v1/map")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -737,7 +736,7 @@ describe("E2E Tests for v1 API Routes", () => {
describe("POST /v1/crawl", () => { describe("POST /v1/crawl", () => {
it.concurrent("should require authorization", async () => { it.concurrent("should require authorization", async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/crawl") .post("/v1/crawl")
.send({ url: "https://firecrawl.dev" }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
@ -761,7 +760,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return an error response with an invalid API key", "should return an error response with an invalid API key",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v1/crawl") .post("/v1/crawl")
.set("Authorization", `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")

47
apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
View File

@ -2,7 +2,6 @@ import request from "supertest";
import { configDotenv } from "dotenv"; import { configDotenv } from "dotenv";
import { import {
ScrapeRequest, ScrapeRequest,
ScrapeResponseRequestTest,
} from "../../controllers/v1/types"; } from "../../controllers/v1/types";
configDotenv(); configDotenv();
@ -13,7 +12,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 403 page", "should return a successful response for a scrape with 403 page",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -40,7 +39,7 @@ describe("E2E Tests for v1 API Routes", () => {
url: E2E_TEST_SERVER_URL, url: E2E_TEST_SERVER_URL,
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -87,7 +86,7 @@ describe("E2E Tests for v1 API Routes", () => {
formats: ["html"], formats: ["html"],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -122,7 +121,7 @@ describe("E2E Tests for v1 API Routes", () => {
formats: ["rawHtml"], formats: ["rawHtml"],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -160,7 +159,7 @@ describe("E2E Tests for v1 API Routes", () => {
headers: { "e2e-header-test": "firecrawl" }, headers: { "e2e-header-test": "firecrawl" },
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -189,7 +188,7 @@ describe("E2E Tests for v1 API Routes", () => {
includeTags: ["#content-1"], includeTags: ["#content-1"],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -221,7 +220,7 @@ describe("E2E Tests for v1 API Routes", () => {
excludeTags: ["#content-1"], excludeTags: ["#content-1"],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -254,7 +253,7 @@ describe("E2E Tests for v1 API Routes", () => {
onlyMainContent: false, onlyMainContent: false,
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -286,7 +285,7 @@ describe("E2E Tests for v1 API Routes", () => {
timeout: 500, timeout: 500,
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -313,7 +312,7 @@ describe("E2E Tests for v1 API Routes", () => {
mobile: true, mobile: true,
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -336,7 +335,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent( it.concurrent(
"should handle 'parsePDF' parameter correctly", "should handle 'parsePDF' parameter correctly",
async () => { async () => {
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -358,7 +357,7 @@ describe("E2E Tests for v1 API Routes", () => {
"h7uKu14adDL6yGfnGf2qycY5uq8kC3OKCWkPxm", "h7uKu14adDL6yGfnGf2qycY5uq8kC3OKCWkPxm",
); );
const responseNoParsePDF: ScrapeResponseRequestTest = await request( const responseNoParsePDF: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -392,7 +391,7 @@ describe("E2E Tests for v1 API Routes", () => {
// } // }
// }; // };
// const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL) // const response: any = await request(FIRECRAWL_API_URL)
// .post("/v1/scrape") // .post("/v1/scrape")
// .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
// .set("Content-Type", "application/json") // .set("Content-Type", "application/json")
@ -411,7 +410,7 @@ describe("E2E Tests for v1 API Routes", () => {
timeout: 120000, timeout: 120000,
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -433,7 +432,7 @@ describe("E2E Tests for v1 API Routes", () => {
timeout: 120000, timeout: 120000,
} as ScrapeRequest; } as ScrapeRequest;
const responseWithSkipTlsVerification: ScrapeResponseRequestTest = const responseWithSkipTlsVerification: any =
await request(FIRECRAWL_API_URL) await request(FIRECRAWL_API_URL)
.post("/v1/scrape") .post("/v1/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@ -462,7 +461,7 @@ describe("E2E Tests for v1 API Routes", () => {
removeBase64Images: true, removeBase64Images: true,
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -494,7 +493,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -527,7 +526,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -570,7 +569,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -620,7 +619,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -658,7 +657,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -693,7 +692,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")
@ -732,7 +731,7 @@ describe("E2E Tests for v1 API Routes", () => {
], ],
} as ScrapeRequest; } as ScrapeRequest;
const response: ScrapeResponseRequestTest = await request( const response: any = await request(
FIRECRAWL_API_URL, FIRECRAWL_API_URL,
) )
.post("/v1/scrape") .post("/v1/scrape")

59
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -1,10 +1,5 @@
import request from "supertest"; import request from "supertest";
import dotenv from "dotenv"; import dotenv from "dotenv";
import {
FirecrawlCrawlResponse,
FirecrawlCrawlStatusResponse,
FirecrawlScrapeResponse,
} from "../../types";
dotenv.config(); dotenv.config();
const TEST_URL = "http://127.0.0.1:3002"; const TEST_URL = "http://127.0.0.1:3002";
@ -28,7 +23,7 @@ describe("E2E Tests for v0 API Routes", () => {
describe("POST /v0/scrape", () => { describe("POST /v0/scrape", () => {
it.concurrent("should require authorization", async () => { it.concurrent("should require authorization", async () => {
const response: FirecrawlScrapeResponse = const response: any =
await request(TEST_URL).post("/v0/scrape"); await request(TEST_URL).post("/v0/scrape");
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
@ -36,7 +31,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return an error response with an invalid API key", "should return an error response with an invalid API key",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -48,7 +43,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with a valid API key", "should return a successful response with a valid API key",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -92,7 +87,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with a valid API key and includeHtml set to true", "should return a successful response with a valid API key and includeHtml set to true",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -118,7 +113,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a valid scrape with PDF file", "should return a successful response for a valid scrape with PDF file",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -141,7 +136,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a valid scrape with PDF file without explicit .pdf extension", "should return a successful response for a valid scrape with PDF file without explicit .pdf extension",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -164,7 +159,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with a valid API key with removeTags option", "should return a successful response with a valid API key with removeTags option",
async () => { async () => {
const responseWithoutRemoveTags: FirecrawlScrapeResponse = const responseWithoutRemoveTags: any =
await request(TEST_URL) await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@ -189,7 +184,7 @@ describe("E2E Tests for v0 API Routes", () => {
"web scraping", "web scraping",
); // strong ); // strong
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -214,7 +209,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 400 page", "should return a successful response for a scrape with 400 page",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -236,7 +231,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 401 page", "should return a successful response for a scrape with 401 page",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -258,7 +253,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 403 page", "should return a successful response for a scrape with 403 page",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -280,7 +275,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 404 page", "should return a successful response for a scrape with 404 page",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -318,7 +313,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response for a scrape with 500 page", "should return a successful response for a scrape with 500 page",
async () => { async () => {
const response: FirecrawlScrapeResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -337,7 +332,7 @@ describe("E2E Tests for v0 API Routes", () => {
describe("POST /v0/crawl", () => { describe("POST /v0/crawl", () => {
it.concurrent("should require authorization", async () => { it.concurrent("should require authorization", async () => {
const response: FirecrawlCrawlResponse = const response: any =
await request(TEST_URL).post("/v0/crawl"); await request(TEST_URL).post("/v0/crawl");
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
@ -345,7 +340,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return an error response with an invalid API key", "should return an error response with an invalid API key",
async () => { async () => {
const response: FirecrawlCrawlResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/crawl") .post("/v0/crawl")
.set("Authorization", `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -357,7 +352,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with a valid API key for crawl", "should return a successful response with a valid API key for crawl",
async () => { async () => {
const response: FirecrawlCrawlResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/crawl") .post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -373,7 +368,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with a valid API key and valid includes option", "should return a successful response with a valid API key and valid includes option",
async () => { async () => {
const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL) const crawlResponse: any = await request(TEST_URL)
.post("/v0/crawl") .post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -385,7 +380,7 @@ describe("E2E Tests for v0 API Routes", () => {
}, },
}); });
let response: FirecrawlCrawlStatusResponse; let response: any;
let isFinished = false; let isFinished = false;
while (!isFinished) { while (!isFinished) {
@ -436,7 +431,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with a valid API key and valid excludes option", "should return a successful response with a valid API key and valid excludes option",
async () => { async () => {
const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL) const crawlResponse: any = await request(TEST_URL)
.post("/v0/crawl") .post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -449,7 +444,7 @@ describe("E2E Tests for v0 API Routes", () => {
}); });
let isFinished = false; let isFinished = false;
let response: FirecrawlCrawlStatusResponse; let response: any;
while (!isFinished) { while (!isFinished) {
response = await request(TEST_URL) response = await request(TEST_URL)
@ -466,7 +461,7 @@ describe("E2E Tests for v0 API Routes", () => {
} }
await new Promise((resolve) => setTimeout(resolve, 1000)); // wait for data to be saved on the database await new Promise((resolve) => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse: FirecrawlCrawlStatusResponse = await request( const completedResponse: any = await request(
TEST_URL, TEST_URL,
) )
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`) .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
@ -486,7 +481,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a successful response with max depth option for a valid crawl job", "should return a successful response with max depth option for a valid crawl job",
async () => { async () => {
const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL) const crawlResponse: any = await request(TEST_URL)
.post("/v0/crawl") .post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -496,7 +491,7 @@ describe("E2E Tests for v0 API Routes", () => {
}); });
expect(crawlResponse.statusCode).toBe(200); expect(crawlResponse.statusCode).toBe(200);
const response: FirecrawlCrawlStatusResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`) .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
@ -514,7 +509,7 @@ describe("E2E Tests for v0 API Routes", () => {
await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
} }
} }
const completedResponse: FirecrawlCrawlStatusResponse = await request( const completedResponse: any = await request(
TEST_URL, TEST_URL,
) )
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`) .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
@ -556,7 +551,7 @@ describe("E2E Tests for v0 API Routes", () => {
describe("POST /v0/crawlWebsitePreview", () => { describe("POST /v0/crawlWebsitePreview", () => {
it.concurrent("should require authorization", async () => { it.concurrent("should require authorization", async () => {
const response: FirecrawlCrawlResponse = await request(TEST_URL).post( const response: any = await request(TEST_URL).post(
"/v0/crawlWebsitePreview", "/v0/crawlWebsitePreview",
); );
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
@ -565,7 +560,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return an error response with an invalid API key", "should return an error response with an invalid API key",
async () => { async () => {
const response: FirecrawlCrawlResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/crawlWebsitePreview") .post("/v0/crawlWebsitePreview")
.set("Authorization", `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
@ -577,7 +572,7 @@ describe("E2E Tests for v0 API Routes", () => {
it.concurrent( it.concurrent(
"should return a timeout error when scraping takes longer than the specified timeout", "should return a timeout error when scraping takes longer than the specified timeout",
async () => { async () => {
const response: FirecrawlCrawlResponse = await request(TEST_URL) const response: any = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")