mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-06-30 00:55:10 +08:00
Nick: temp file fixes
This commit is contained in:
Nicolas
parent
ac6650e488
commit
9759f18725
@ -2,7 +2,6 @@ import request from "supertest";
|
||||
import { configDotenv } from "dotenv";
|
||||
import {
|
||||
ScrapeRequestInput,
|
||||
ScrapeResponseRequestTest,
|
||||
} from "../../controllers/v1/types";
|
||||
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
|
||||
|
||||
@ -20,7 +19,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
|
||||
describe("GET /is-production", () => {
|
||||
it.concurrent("should return the production status", async () => {
|
||||
const response: ScrapeResponseRequestTest =
|
||||
const response: any =
|
||||
await request(TEST_URL).get("/is-production");
|
||||
|
||||
console.log(
|
||||
@ -39,7 +38,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
|
||||
describe("POST /v1/scrape", () => {
|
||||
it.concurrent("should require authorization", async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.send({ url: "https://firecrawl.dev" });
|
||||
|
||||
@ -64,7 +63,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return an error response with an invalid API key",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer invalid-api-key`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -80,7 +79,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
url: "https://roastmywebsite.ai",
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -132,7 +131,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
url: "https://arxiv.org/abs/2410.04840",
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -189,7 +188,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
formats: ["markdown", "html"],
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -217,7 +216,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",
|
||||
// formats: ["markdown", "html"],
|
||||
};
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -245,7 +244,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
const scrapeRequest: ScrapeRequestInput = {
|
||||
url: "https://arxiv.org/pdf/astro-ph/9301001",
|
||||
};
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -275,7 +274,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
url: "https://www.scrapethissite.com/",
|
||||
onlyMainContent: false, // default is true
|
||||
};
|
||||
const responseWithoutRemoveTags: ScrapeResponseRequestTest =
|
||||
const responseWithoutRemoveTags: any =
|
||||
await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
@ -302,7 +301,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
excludeTags: [".nav", "#footer", "strong"],
|
||||
onlyMainContent: false, // default is true
|
||||
};
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -325,7 +324,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 400 page",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -347,7 +346,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 401 page",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -368,7 +367,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
|
||||
// Removed it as we want to retry fallback to the next scraper
|
||||
// it.concurrent('should return a successful response for a scrape with 403 page', async () => {
|
||||
// const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
// const response: any = await request(TEST_URL)
|
||||
// .post('/v1/scrape')
|
||||
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
|
||||
// .set('Content-Type', 'application/json')
|
||||
@ -388,7 +387,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 404 page",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -408,7 +407,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
);
|
||||
|
||||
// it.concurrent('should return a successful response for a scrape with 405 page', async () => {
|
||||
// const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
// const response: any = await request(TEST_URL)
|
||||
// .post('/v1/scrape')
|
||||
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
|
||||
// .set('Content-Type', 'application/json')
|
||||
@ -426,7 +425,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
// }, 60000);
|
||||
|
||||
// it.concurrent('should return a successful response for a scrape with 500 page', async () => {
|
||||
// const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
// const response: any = await request(TEST_URL)
|
||||
// .post('/v1/scrape')
|
||||
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
|
||||
// .set('Content-Type', 'application/json')
|
||||
@ -446,7 +445,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a timeout error when scraping takes longer than the specified timeout",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -465,7 +464,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
formats: ["html", "rawHtml"],
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -497,7 +496,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
waitFor: 8000,
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -528,7 +527,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
formats: ["links"],
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -553,7 +552,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
|
||||
describe("POST /v1/map", () => {
|
||||
it.concurrent("should require authorization", async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.send({ url: "https://firecrawl.dev" });
|
||||
expect(response.statusCode).toBe(401);
|
||||
@ -562,7 +561,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return an error response with an invalid API key",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer invalid-api-key`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -578,7 +577,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
url: "https://roastmywebsite.ai",
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -604,7 +603,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
search: "pricing",
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -632,7 +631,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
includeSubdomains: true,
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -664,7 +663,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
includeSubdomains: true,
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -697,7 +696,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
includeSubdomains: false,
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -723,7 +722,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
search: "test",
|
||||
};
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/map")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -737,7 +736,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
|
||||
describe("POST /v1/crawl", () => {
|
||||
it.concurrent("should require authorization", async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/crawl")
|
||||
.send({ url: "https://firecrawl.dev" });
|
||||
expect(response.statusCode).toBe(401);
|
||||
@ -761,7 +760,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return an error response with an invalid API key",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v1/crawl")
|
||||
.set("Authorization", `Bearer invalid-api-key`)
|
||||
.set("Content-Type", "application/json")
|
||||
|
||||
@ -2,7 +2,6 @@ import request from "supertest";
|
||||
import { configDotenv } from "dotenv";
|
||||
import {
|
||||
ScrapeRequest,
|
||||
ScrapeResponseRequestTest,
|
||||
} from "../../controllers/v1/types";
|
||||
|
||||
configDotenv();
|
||||
@ -13,7 +12,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 403 page",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -40,7 +39,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
url: E2E_TEST_SERVER_URL,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -87,7 +86,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
formats: ["html"],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -122,7 +121,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
formats: ["rawHtml"],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -160,7 +159,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
headers: { "e2e-header-test": "firecrawl" },
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -189,7 +188,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
includeTags: ["#content-1"],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -221,7 +220,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
excludeTags: ["#content-1"],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -254,7 +253,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
onlyMainContent: false,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -286,7 +285,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
timeout: 500,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -313,7 +312,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
mobile: true,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -336,7 +335,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should handle 'parsePDF' parameter correctly",
|
||||
async () => {
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -358,7 +357,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
"h7uKu14adDL6yGfnGf2qycY5uq8kC3OKCWkPxm",
|
||||
);
|
||||
|
||||
const responseNoParsePDF: ScrapeResponseRequestTest = await request(
|
||||
const responseNoParsePDF: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -392,7 +391,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
// }
|
||||
// };
|
||||
|
||||
// const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
|
||||
// const response: any = await request(FIRECRAWL_API_URL)
|
||||
// .post("/v1/scrape")
|
||||
// .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
// .set("Content-Type", "application/json")
|
||||
@ -411,7 +410,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
timeout: 120000,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -433,7 +432,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
timeout: 120000,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const responseWithSkipTlsVerification: ScrapeResponseRequestTest =
|
||||
const responseWithSkipTlsVerification: any =
|
||||
await request(FIRECRAWL_API_URL)
|
||||
.post("/v1/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
@ -462,7 +461,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
removeBase64Images: true,
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -494,7 +493,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -527,7 +526,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -570,7 +569,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -620,7 +619,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -658,7 +657,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -693,7 +692,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
@ -732,7 +731,7 @@ describe("E2E Tests for v1 API Routes", () => {
|
||||
],
|
||||
} as ScrapeRequest;
|
||||
|
||||
const response: ScrapeResponseRequestTest = await request(
|
||||
const response: any = await request(
|
||||
FIRECRAWL_API_URL,
|
||||
)
|
||||
.post("/v1/scrape")
|
||||
|
||||
@ -1,10 +1,5 @@
|
||||
import request from "supertest";
|
||||
import dotenv from "dotenv";
|
||||
import {
|
||||
FirecrawlCrawlResponse,
|
||||
FirecrawlCrawlStatusResponse,
|
||||
FirecrawlScrapeResponse,
|
||||
} from "../../types";
|
||||
|
||||
dotenv.config();
|
||||
const TEST_URL = "http://127.0.0.1:3002";
|
||||
@ -28,7 +23,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
|
||||
describe("POST /v0/scrape", () => {
|
||||
it.concurrent("should require authorization", async () => {
|
||||
const response: FirecrawlScrapeResponse =
|
||||
const response: any =
|
||||
await request(TEST_URL).post("/v0/scrape");
|
||||
expect(response.statusCode).toBe(401);
|
||||
});
|
||||
@ -36,7 +31,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return an error response with an invalid API key",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer invalid-api-key`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -48,7 +43,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with a valid API key",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -92,7 +87,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with a valid API key and includeHtml set to true",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -118,7 +113,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a valid scrape with PDF file",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -141,7 +136,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a valid scrape with PDF file without explicit .pdf extension",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -164,7 +159,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with a valid API key with removeTags option",
|
||||
async () => {
|
||||
const responseWithoutRemoveTags: FirecrawlScrapeResponse =
|
||||
const responseWithoutRemoveTags: any =
|
||||
await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
@ -189,7 +184,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
"web scraping",
|
||||
); // strong
|
||||
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -214,7 +209,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 400 page",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -236,7 +231,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 401 page",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -258,7 +253,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 403 page",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -280,7 +275,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 404 page",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -318,7 +313,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response for a scrape with 500 page",
|
||||
async () => {
|
||||
const response: FirecrawlScrapeResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -337,7 +332,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
|
||||
describe("POST /v0/crawl", () => {
|
||||
it.concurrent("should require authorization", async () => {
|
||||
const response: FirecrawlCrawlResponse =
|
||||
const response: any =
|
||||
await request(TEST_URL).post("/v0/crawl");
|
||||
expect(response.statusCode).toBe(401);
|
||||
});
|
||||
@ -345,7 +340,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return an error response with an invalid API key",
|
||||
async () => {
|
||||
const response: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/crawl")
|
||||
.set("Authorization", `Bearer invalid-api-key`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -357,7 +352,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with a valid API key for crawl",
|
||||
async () => {
|
||||
const response: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/crawl")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -373,7 +368,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with a valid API key and valid includes option",
|
||||
async () => {
|
||||
const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const crawlResponse: any = await request(TEST_URL)
|
||||
.post("/v0/crawl")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -385,7 +380,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
},
|
||||
});
|
||||
|
||||
let response: FirecrawlCrawlStatusResponse;
|
||||
let response: any;
|
||||
let isFinished = false;
|
||||
|
||||
while (!isFinished) {
|
||||
@ -436,7 +431,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with a valid API key and valid excludes option",
|
||||
async () => {
|
||||
const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const crawlResponse: any = await request(TEST_URL)
|
||||
.post("/v0/crawl")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -449,7 +444,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
});
|
||||
|
||||
let isFinished = false;
|
||||
let response: FirecrawlCrawlStatusResponse;
|
||||
let response: any;
|
||||
|
||||
while (!isFinished) {
|
||||
response = await request(TEST_URL)
|
||||
@ -466,7 +461,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000)); // wait for data to be saved on the database
|
||||
const completedResponse: FirecrawlCrawlStatusResponse = await request(
|
||||
const completedResponse: any = await request(
|
||||
TEST_URL,
|
||||
)
|
||||
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
|
||||
@ -486,7 +481,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a successful response with max depth option for a valid crawl job",
|
||||
async () => {
|
||||
const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const crawlResponse: any = await request(TEST_URL)
|
||||
.post("/v0/crawl")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -496,7 +491,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
});
|
||||
expect(crawlResponse.statusCode).toBe(200);
|
||||
|
||||
const response: FirecrawlCrawlStatusResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
|
||||
expect(response.statusCode).toBe(200);
|
||||
@ -514,7 +509,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
|
||||
}
|
||||
}
|
||||
const completedResponse: FirecrawlCrawlStatusResponse = await request(
|
||||
const completedResponse: any = await request(
|
||||
TEST_URL,
|
||||
)
|
||||
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
|
||||
@ -556,7 +551,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
|
||||
describe("POST /v0/crawlWebsitePreview", () => {
|
||||
it.concurrent("should require authorization", async () => {
|
||||
const response: FirecrawlCrawlResponse = await request(TEST_URL).post(
|
||||
const response: any = await request(TEST_URL).post(
|
||||
"/v0/crawlWebsitePreview",
|
||||
);
|
||||
expect(response.statusCode).toBe(401);
|
||||
@ -565,7 +560,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return an error response with an invalid API key",
|
||||
async () => {
|
||||
const response: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/crawlWebsitePreview")
|
||||
.set("Authorization", `Bearer invalid-api-key`)
|
||||
.set("Content-Type", "application/json")
|
||||
@ -577,7 +572,7 @@ describe("E2E Tests for v0 API Routes", () => {
|
||||
it.concurrent(
|
||||
"should return a timeout error when scraping takes longer than the specified timeout",
|
||||
async () => {
|
||||
const response: FirecrawlCrawlResponse = await request(TEST_URL)
|
||||
const response: any = await request(TEST_URL)
|
||||
.post("/v0/scrape")
|
||||
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
|
||||
.set("Content-Type", "application/json")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user