AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 03:40:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #393 from mendableai/mog/job-stuck-fix

Browse Source 
fix(bull): requeue jobs after restart
This commit is contained in:
Nicolas 2024-07-12 10:43:02 -04:00 committed by GitHub
commit c07a1912e9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 39 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
apps/api/src/index.ts
View File

@ -119,23 +119,36 @@ if (cluster.isMaster) {
app.post(`/admin/${process.env.BULL_AUTH_KEY}/shutdown`, async (req, res) => { app.post(`/admin/${process.env.BULL_AUTH_KEY}/shutdown`, async (req, res) => {
return res.status(200).json({ ok: true }); return res.status(200).json({ ok: true });
try {
console.log("Gracefully shutting down...");
await getWebScraperQueue().pause(false, true);
res.json({ ok: true });
} catch (error) {
console.error(error);
return res.status(500).json({ error: error.message });
}
});
app.post(`/admin/${process.env.BULL_AUTH_KEY}/unpause`, async (req, res) => {
try { try {
const wsq = getWebScraperQueue(); const wsq = getWebScraperQueue();
console.log("Gracefully shutting down...");
await wsq.pause(false, true);
const jobs = await wsq.getActive(); const jobs = await wsq.getActive();
console.log("Requeueing", jobs.length, "jobs...");
if (jobs.length > 0) { if (jobs.length > 0) {
console.log(" Removing", jobs.length, "jobs..."); console.log(" Removing", jobs.length, "jobs...");
await Promise.all(jobs.map(async x => { await Promise.all(jobs.map(async x => {
try {
await wsq.client.del(await x.lockKey()); await wsq.client.del(await x.lockKey());
await x.takeLock(); await x.takeLock();
await x.moveToFailed({ message: "interrupted" }); await x.moveToFailed({ message: "interrupted" });
await x.remove(); await x.remove();
} catch (e) {
console.warn("Failed to remove job", x.id, e);
}
})); }));
console.log(" Re-adding", jobs.length, "jobs..."); console.log(" Re-adding", jobs.length, "jobs...");
@ -147,22 +160,16 @@ if (cluster.isMaster) {
}))); })));
console.log(" Done!"); console.log(" Done!");
res.json({ ok: true });
} }
await getWebScraperQueue().resume(false);
res.json({ ok: true });
} catch (error) { } catch (error) {
console.error(error); console.error(error);
return res.status(500).json({ error: error.message }); return res.status(500).json({ error: error.message });
} }
}); });
app.post(`/admin/${process.env.BULL_AUTH_KEY}/unpause`, async (req, res) => {
return res.status(200).json({ ok: true });
await getWebScraperQueue().resume(false);
res.json({ ok: true });
});
app.get(`/serverHealthCheck`, async (req, res) => { app.get(`/serverHealthCheck`, async (req, res) => {
try { try {
const webScraperQueue = getWebScraperQueue(); const webScraperQueue = getWebScraperQueue();