diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 9c41073a..d204bcd4 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -104,7 +104,7 @@ export async function runWebScraper({ return { url: doc.metadata.sourceURL }; } }) - : docs.filter((doc) => doc.content.trim().length > 0); + : docs; const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + bull_job_id); diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 3701d40e..c5af5fab 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -52,7 +52,6 @@ const processJobInternal = async (token: string, job: Job) => { try { const result = await processJob(job, token); const jobState = await job.getState(); - console.log("done", job, jobState, result); if (jobState !== "completed" && jobState !== "failed") { await job.moveToCompleted(result.docs, token, false); }