AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 08:26:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(runWebScraper): don't filter empty docs

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-07 21:00:22 +02:00
parent 55ec96c23f
commit 920b7f2f44
2 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/main/runWebScraper.ts
View File

@ -104,7 +104,7 @@ export async function runWebScraper({
return { url: doc.metadata.sourceURL }; return { url: doc.metadata.sourceURL };
} }
}) })
: docs.filter((doc) => doc.content.trim().length > 0); : docs;
const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + bull_job_id); const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + bull_job_id);

1
apps/api/src/services/queue-worker.ts
View File

@ -52,7 +52,6 @@ const processJobInternal = async (token: string, job: Job) => {
try { try {
const result = await processJob(job, token); const result = await processJob(job, token);
const jobState = await job.getState(); const jobState = await job.getState();
console.log("done", job, jobState, result);
if (jobState !== "completed" && jobState !== "failed") { if (jobState !== "completed" && jobState !== "failed") {
await job.moveToCompleted(result.docs, token, false); await job.moveToCompleted(result.docs, token, false);
} }