AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 02:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawl-status): handle failed jobs (oops)

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-04 17:52:47 +01:00
parent 712a138404
commit d8613899e3
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -99,8 +99,14 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
// both loops will break once we cross the byte counter
for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {
const job = jobs[ii];
const state = await job.getState();
if (state === "failed") {
continue;
}
if (job.returnvalue === undefined) {
logger.warn("Job was considered done, but returnvalue is undefined!", { jobId: job.id });
logger.warn("Job was considered done, but returnvalue is undefined!", { jobId: job.id, state });
continue;
}
doneJobs.push(job);
@ -113,7 +119,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
doneJobs.splice(doneJobs.length - 1, 1);
}
} else {
doneJobs = await getJobs(doneJobsOrder);
doneJobs = (await Promise.all((await getJobs(doneJobsOrder)).map(async x => (await x.getState()) === "failed" ? null : x))).filter(x => x !== null);
}
const data = doneJobs.map(x => x.returnvalue);