AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 23:28:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrape): record job result in span

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-22 16:00:13 +02:00
parent 5ca36fe9fc
commit 6d92b8524d
1 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
apps/api/src/controllers/scrape.ts
View File

@ -49,18 +49,28 @@ export async function scrapeHelper(
}, {}, jobId);
let doc;
try {
doc = (await job.waitUntilFinished(scrapeQueueEvents, timeout))[0]; //60 seconds timeout
} catch (e) {
if (e instanceof Error && e.message.startsWith("Job wait")) {
return {
success: false,
error: "Request timed out",
returnCode: 408,
const err = await Sentry.startSpanManual({ name: "Wait for job to finish", op: "bullmq.wait", attributes: { job: jobId } }, async (span) => {
try {
doc = (await job.waitUntilFinished(scrapeQueueEvents, timeout))[0]
} catch (e) {
if (e instanceof Error && e.message.startsWith("Job wait")) {
span.setAttribute("timedOut", true).end();
return {
success: false,
error: "Request timed out",
returnCode: 408,
}
} else {
throw e;
}
} else {
throw e;
}
span.setAttribute("result", JSON.stringify(doc)).end();
return null;
});
if (err !== null) {
return err;
}
await job.remove();