AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 13:18:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(queue-worker): errored job logging

Browse Source
This commit is contained in:
Gergő Móricz 2025-01-09 18:48:47 +01:00
parent 97bf54214f
commit 2849ce2f13
1 changed files with 24 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
apps/api/src/services/queue-worker.ts
View File

@ -989,16 +989,30 @@ async function processJob(job: Job & { id: string }, token: string) {
job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
);
}
// if (job.data.v1) {
// callWebhook(
// job.data.team_id,
// job.id as string,
// [],
// job.data.webhook,
// job.data.v1,
// "crawl.failed"
// );
// }
logger.debug("Logging job to DB...");
await logJob(
{
job_id: job.id as string,
success: false,
message:
typeof error === "string"
? error
: (error.message ??
"Something went wrong... Contact help@mendable.ai"),
num_docs: 0,
docs: [],
time_taken: 0,
team_id: job.data.team_id,
mode: job.data.mode,
url: job.data.url,
crawlerOptions: job.data.crawlerOptions,
scrapeOptions: job.data.scrapeOptions,
origin: job.data.origin,
crawl_id: job.data.crawl_id,
},
true,
);
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
@ -1010,30 +1024,6 @@ async function processJob(job: Job & { id: string }, token: string) {
normalizeURL(job.data.url, sc),
);
logger.debug("Logging job to DB...");
await logJob(
{
job_id: job.id as string,
success: false,
message:
typeof error === "string"
? error
: (error.message ??
"Something went wrong... Contact help@mendable.ai"),
num_docs: 0,
docs: [],
time_taken: 0,
team_id: job.data.team_id,
mode: job.data.mode,
url: job.data.url,
crawlerOptions: sc.crawlerOptions,
scrapeOptions: job.data.scrapeOptions,
origin: job.data.origin,
crawl_id: job.data.crawl_id,
},
true,
);
await finishCrawlIfNeeded(job, sc);
// await logJob({