AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 09:36:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrape): record job result in span

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-22 16:00:13 +02:00
parent 5ca36fe9fc
commit 6d92b8524d
1 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
apps/api/src/controllers/scrape.ts
View File

@ -49,18 +49,28 @@ export async function scrapeHelper(
}, {}, jobId); }, {}, jobId);
let doc; let doc;
try {
doc = (await job.waitUntilFinished(scrapeQueueEvents, timeout))[0]; //60 seconds timeout const err = await Sentry.startSpanManual({ name: "Wait for job to finish", op: "bullmq.wait", attributes: { job: jobId } }, async (span) => {
} catch (e) { try {
if (e instanceof Error && e.message.startsWith("Job wait")) { doc = (await job.waitUntilFinished(scrapeQueueEvents, timeout))[0]
return { } catch (e) {
success: false, if (e instanceof Error && e.message.startsWith("Job wait")) {
error: "Request timed out", span.setAttribute("timedOut", true).end();
returnCode: 408, return {
success: false,
error: "Request timed out",
returnCode: 408,
}
} else {
throw e;
} }
} else {
throw e;
} }
span.setAttribute("result", JSON.stringify(doc)).end();
return null;
});
if (err !== null) {
return err;
} }
await job.remove(); await job.remove();