AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 20:56:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(queue-worker): do not kill crawl on one-page error

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-12 22:53:29 +01:00
parent 16e850288c
commit 93ac20f930
1 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
apps/api/src/services/queue-worker.ts
View File

@ -568,24 +568,24 @@ async function processJob(job: Job & { id: string }, token: string) {
crawl_id: job.data.crawl_id, crawl_id: job.data.crawl_id,
}); });
await logJob({ // await logJob({
job_id: job.data.crawl_id, // job_id: job.data.crawl_id,
success: false, // success: false,
message: // message:
typeof error === "string" // typeof error === "string"
? error // ? error
: error.message ?? // : error.message ??
"Something went wrong... Contact help@mendable.ai", // "Something went wrong... Contact help@mendable.ai",
num_docs: 0, // num_docs: 0,
docs: [], // docs: [],
time_taken: 0, // time_taken: 0,
team_id: job.data.team_id, // team_id: job.data.team_id,
mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape", // mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
url: sc ? sc.originUrl ?? job.data.url : job.data.url, // url: sc ? sc.originUrl ?? job.data.url : job.data.url,
crawlerOptions: sc ? sc.crawlerOptions : undefined, // crawlerOptions: sc ? sc.crawlerOptions : undefined,
scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions, // scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions,
origin: job.data.origin, // origin: job.data.origin,
}); // });
} }
// done(null, data); // done(null, data);
return data; return data;