AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 02:16:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(queue-worker): manually renew lock (testing)

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-07 14:35:20 +02:00
parent 8216266d16
commit 7bb922071c
2 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/index.ts
View File

@ -94,7 +94,7 @@ export class WebScraperDataProvider {
const jobStatus = await job.getState();
if (jobStatus === "failed") {
Logger.info(
"Job has failed or has been cancelled by the user. Stopping the job..."
"Job " + job.id + " has failed or has been cancelled by the user. Stopping the job..."
);
return [] as Document[];
}

9
apps/api/src/services/queue-worker.ts
View File

@ -22,6 +22,11 @@ const wsq = getWebScraperQueue();
async function processJob(job: Job, done) {
Logger.debug(`🐂 Worker taking job ${job.id}`);
const lockInterval = setInterval(() => {
Logger.debug(`🐂 Renewing lock for ${job.id}`);
job.extendLock(60000);
}, 15000);
try {
job.progress({
current: 1,
@ -62,6 +67,7 @@ async function processJob(job: Job, done) {
origin: job.data.origin,
});
Logger.debug(`🐂 Job done ${job.id}`);
clearInterval(lockInterval);
done(null, data);
} catch (error) {
Logger.error(`🐂 Job errored ${job.id} - ${error}`);
@ -108,8 +114,9 @@ async function processJob(job: Job, done) {
pageOptions: job.data.pageOptions,
origin: job.data.origin,
});
clearInterval(lockInterval);
done(null, data);
}
}
}
wsq.process(