diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 84c6d42e..de8f5567 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -997,6 +997,19 @@ async function processJob(job: Job & { id: string }, token: string) { logger.info(`🐂 Job done ${job.id}`); return data; } catch (error) { + if (job.data.crawl_id) { + const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl; + + logger.debug("Declaring job as done..."); + await addCrawlJobDone(job.data.crawl_id, job.id, false); + await redisConnection.srem( + "crawl:" + job.data.crawl_id + ":visited_unique", + normalizeURL(job.data.url, sc), + ); + + await finishCrawlIfNeeded(job, sc); + } + const isEarlyTimeout = error instanceof Error && error.message === "timeout"; const isCancelled = @@ -1076,39 +1089,6 @@ async function processJob(job: Job & { id: string }, token: string) { }, true, ); - - if (job.data.crawl_id) { - const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl; - - logger.debug("Declaring job as done..."); - await addCrawlJobDone(job.data.crawl_id, job.id, false); - await redisConnection.srem( - "crawl:" + job.data.crawl_id + ":visited_unique", - normalizeURL(job.data.url, sc), - ); - - await finishCrawlIfNeeded(job, sc); - - // await logJob({ - // job_id: job.data.crawl_id, - // success: false, - // message: - // typeof error === "string" - // ? error - // : error.message ?? - // "Something went wrong... Contact help@mendable.ai", - // num_docs: 0, - // docs: [], - // time_taken: 0, - // team_id: job.data.team_id, - // mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape", - // url: sc ? sc.originUrl ?? job.data.url : job.data.url, - // crawlerOptions: sc ? sc.crawlerOptions : undefined, - // scrapeOptions: sc ? sc.scrapeOptions : job.data.scrapeOptions, - // origin: job.data.origin, - // }); - } - // done(null, data); return data; } }