AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 18:40:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

added check for object and trycatch as workaround for 502s

Browse Source
This commit is contained in:
rafaelmmiller 2024-12-13 18:33:39 -03:00
parent 6b17a53d4b
commit bdbc05a4c7
2 changed files with 39 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "1.9.3", "version": "1.9.4",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.ts", "types": "dist/index.d.ts",

72
apps/js-sdk/firecrawl/src/index.ts
View File

@ -462,7 +462,7 @@ export default class FirecrawlApp {
let statusData = response.data let statusData = response.data
if ("data" in statusData) { if ("data" in statusData) {
let data = statusData.data; let data = statusData.data;
while ('next' in statusData) { while (typeof statusData === 'object' && 'next' in statusData) {
statusData = (await this.getRequest(statusData.next, headers)).data; statusData = (await this.getRequest(statusData.next, headers)).data;
data = data.concat(statusData.data); data = data.concat(statusData.data);
} }
@ -691,7 +691,7 @@ export default class FirecrawlApp {
let statusData = response.data let statusData = response.data
if ("data" in statusData) { if ("data" in statusData) {
let data = statusData.data; let data = statusData.data;
while ('next' in statusData) { while (typeof statusData === 'object' && 'next' in statusData) {
statusData = (await this.getRequest(statusData.next, headers)).data; statusData = (await this.getRequest(statusData.next, headers)).data;
data = data.concat(statusData.data); data = data.concat(statusData.data);
} }
@ -850,42 +850,46 @@ export default class FirecrawlApp {
headers: AxiosRequestHeaders, headers: AxiosRequestHeaders,
checkInterval: number checkInterval: number
): Promise<CrawlStatusResponse | ErrorResponse> { ): Promise<CrawlStatusResponse | ErrorResponse> {
while (true) { try {
let statusResponse: AxiosResponse = await this.getRequest( while (true) {
`${this.apiUrl}/v1/crawl/${id}`, let statusResponse: AxiosResponse = await this.getRequest(
headers `${this.apiUrl}/v1/crawl/${id}`,
); headers
if (statusResponse.status === 200) { );
let statusData = statusResponse.data; if (statusResponse.status === 200) {
if (statusData.status === "completed") { let statusData = statusResponse.data;
if ("data" in statusData) { if (statusData.status === "completed") {
let data = statusData.data; if ("data" in statusData) {
while ('next' in statusData) { let data = statusData.data;
statusResponse = await this.getRequest(statusData.next, headers); while (typeof statusData === 'object' && 'next' in statusData) {
statusData = statusResponse.data; statusResponse = await this.getRequest(statusData.next, headers);
data = data.concat(statusData.data); statusData = statusResponse.data;
data = data.concat(statusData.data);
}
statusData.data = data;
return statusData;
} else {
throw new FirecrawlError("Crawl job completed but no data was returned", 500);
} }
statusData.data = data; } else if (
return statusData; ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
} else { ) {
throw new FirecrawlError("Crawl job completed but no data was returned", 500); checkInterval = Math.max(checkInterval, 2);
} await new Promise((resolve) =>
} else if ( setTimeout(resolve, checkInterval * 1000)
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status) );
) { } else {
checkInterval = Math.max(checkInterval, 2); throw new FirecrawlError(
await new Promise((resolve) => `Crawl job failed or was stopped. Status: ${statusData.status}`,
setTimeout(resolve, checkInterval * 1000) 500
); );
}
} else { } else {
throw new FirecrawlError( this.handleError(statusResponse, "check crawl status");
`Crawl job failed or was stopped. Status: ${statusData.status}`,
500
);
} }
} else {
this.handleError(statusResponse, "check crawl status");
} }
} catch (error: any) {
throw new FirecrawlError(error, 500);
} }
} }