AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 15:30:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(runWebScraper): don't filter empty docs

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-07 21:00:22 +02:00
parent 55ec96c23f
commit 920b7f2f44
2 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/main/runWebScraper.ts
View File

@ -104,7 +104,7 @@ export async function runWebScraper({
return { url: doc.metadata.sourceURL };
}
})
: docs.filter((doc) => doc.content.trim().length > 0);
: docs;
const isCancelled = await (await getWebScraperQueue().client).exists("cancelled:" + bull_job_id);

1
apps/api/src/services/queue-worker.ts
View File

@ -52,7 +52,6 @@ const processJobInternal = async (token: string, job: Job) => {
try {
const result = await processJob(job, token);
const jobState = await job.getState();
console.log("done", job, jobState, result);
if (jobState !== "completed" && jobState !== "failed") {
await job.moveToCompleted(result.docs, token, false);
}