AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 02:49:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(sdk): js next pagination

Browse Source
This commit is contained in:
rafaelsideguide 2024-09-03 11:57:19 -03:00
parent 411d7f31c5
commit 6ccc22ba2f
2 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "1.2.1", "version": "1.2.2",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "build/cjs/index.js", "main": "build/cjs/index.js",
"types": "types/index.d.ts", "types": "types/index.d.ts",

13
apps/js-sdk/firecrawl/src/index.ts
View File

@ -454,20 +454,27 @@ export default class FirecrawlApp {
checkInterval: number checkInterval: number
): Promise<CrawlStatusResponse> { ): Promise<CrawlStatusResponse> {
while (true) { while (true) {
const statusResponse: AxiosResponse = await this.getRequest( let statusResponse: AxiosResponse = await this.getRequest(
`${this.apiUrl}/v1/crawl/${id}`, `${this.apiUrl}/v1/crawl/${id}`,
headers headers
); );
if (statusResponse.status === 200) { if (statusResponse.status === 200) {
const statusData = statusResponse.data; let statusData = statusResponse.data;
if (statusData.status === "completed") { if (statusData.status === "completed") {
if ("data" in statusData) { if ("data" in statusData) {
let data = statusData.data;
while ('next' in statusData) {
statusResponse = await this.getRequest(statusData.next, headers);
statusData = statusResponse.data;
data = data.concat(statusData.data);
}
statusData.data = data;
return statusData; return statusData;
} else { } else {
throw new Error("Crawl job completed but no data was returned"); throw new Error("Crawl job completed but no data was returned");
} }
} else if ( } else if (
["active", "paused", "pending", "queued", "scraping"].includes(statusData.status) ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
) { ) {
checkInterval = Math.max(checkInterval, 2); checkInterval = Math.max(checkInterval, 2);
await new Promise((resolve) => await new Promise((resolve) =>