AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-08 19:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(queue-worker): proactive job cancel

Browse Source
This commit is contained in:
Gergő Móricz 2025-01-15 19:02:20 +01:00
parent ec039dcb8f
commit cbe67d89a5
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/services/queue-worker.ts
View File

@ -739,6 +739,13 @@ async function processJob(job: Job & { id: string }, token: string) {
}); });
const start = Date.now(); const start = Date.now();
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
if (sc && sc.cancelled) {
throw new Error("Parent crawl/batch scrape was cancelled");
}
}
const pipeline = await Promise.race([ const pipeline = await Promise.race([
startWebScraperPipeline({ startWebScraperPipeline({
job, job,
@ -983,11 +990,15 @@ async function processJob(job: Job & { id: string }, token: string) {
} catch (error) { } catch (error) {
const isEarlyTimeout = const isEarlyTimeout =
error instanceof Error && error.message === "timeout"; error instanceof Error && error.message === "timeout";
const isCancelled =
error instanceof Error && error.message === "Parent crawl/batch scrape was cancelled";
if (isEarlyTimeout) { if (isEarlyTimeout) {
logger.error(`🐂 Job timed out ${job.id}`); logger.error(`🐂 Job timed out ${job.id}`);
} else if (error instanceof RacedRedirectError) { } else if (error instanceof RacedRedirectError) {
logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`); logger.warn(`🐂 Job got redirect raced ${job.id}, silently failing`);
} else if (isCancelled) {
logger.warn(`🐂 Job got cancelled, silently failing`);
} else { } else {
logger.error(`🐂 Job errored ${job.id} - ${error}`, { error }); logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });