From 920b7f2f44268967ca4223083e1d6cd8ce6cfc17 Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Wed, 7 Aug 2024 21:00:22 +0200 Subject: [PATCH] fix(runWebScraper): don't filter empty docs --- apps/api/src/main/runWebScraper.ts | 2 +- apps/api/src/services/queue-worker.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 9c41073a..d204bcd4 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -104,7 +104,7 @@ export async function runWebScraper({ return { url: doc.metadata.sourceURL }; } }) - : docs.filter((doc) => doc.content.trim().length > 0); + : docs; const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + bull_job_id); diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 3701d40e..c5af5fab 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -52,7 +52,6 @@ const processJobInternal = async (token: string, job: Job) => { try { const result = await processJob(job, token); const jobState = await job.getState(); - console.log("done", job, jobState, result); if (jobState !== "completed" && jobState !== "failed") { await job.moveToCompleted(result.docs, token, false); }