AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 20:26:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-10-03 17:20:57 -03:00
parent 82551bb6bc
commit ddd774ed68
2 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
View File

@ -333,23 +333,23 @@ describe("E2E Tests for v1 API Routes", () => {
// expect(response.body.data.metadata.statusCode).toBe(405);
// }, 60000);
it.concurrent('should return a successful response for a scrape with 500 page', async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL)
.post('/v1/scrape')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json')
.send({ url: 'https://httpstat.us/500' });
await new Promise((r) => setTimeout(r, 5000));
// it.concurrent('should return a successful response for a scrape with 500 page', async () => {
// const response: ScrapeResponseRequestTest = await request(TEST_URL)
// .post('/v1/scrape')
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
// .set('Content-Type', 'application/json')
// .send({ url: 'https://httpstat.us/500' });
// await new Promise((r) => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('data');
if (!("data" in response.body)) {
throw new Error("Expected response body to have 'data' property");
}
expect(response.body.data).toHaveProperty('markdown');
expect(response.body.data).toHaveProperty('metadata');
expect(response.body.data.metadata.statusCode).toBe(500);
}, 60000);
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty('data');
// if (!("data" in response.body)) {
// throw new Error("Expected response body to have 'data' property");
// }
// expect(response.body.data).toHaveProperty('markdown');
// expect(response.body.data).toHaveProperty('metadata');
// expect(response.body.data.metadata.statusCode).toBe(500);
// }, 60000);
it.concurrent("should return a timeout error when scraping takes longer than the specified timeout", async () => {
const response: ScrapeResponseRequestTest = await request(TEST_URL)
@ -681,7 +681,7 @@ describe("POST /v1/crawl", () => {
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
limit: 20,
limit: 40,
includePaths: ["blog/*"],
});
@ -737,7 +737,7 @@ describe("POST /v1/crawl", () => {
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
limit: 20,
limit: 40,
excludePaths: ["blog/*"],
});
@ -929,11 +929,11 @@ describe("GET /v1/crawl/:jobId", () => {
.post("/v1/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://docs.tatum.io", limit: 200 });
.send({ url: "https://docs.firecrawl.dev", limit: 10 });
expect(crawlResponse.statusCode).toBe(200);
await new Promise((r) => setTimeout(r, 5000));
await new Promise((r) => setTimeout(r, 10000));
const responseCancel = await request(TEST_URL)
.delete(`/v1/crawl/${crawlResponse.body.id}`)

2
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -425,7 +425,7 @@ export async function scrapSingleUrl(
Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);
break;
}
if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) {
if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400 || pageStatusCode == 401)) {
Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`);
break;
}