AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 03:09:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(unvisitedUrls): filter with crawler

Browse Source 
Fixes #1410
This commit is contained in:
Gergő Móricz 2025-04-04 22:12:59 +02:00
parent 6bed5eca50
commit 570809aa59
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
apps/api/src/services/queue-worker.ts
View File

@ -118,7 +118,19 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
const lastUrlsSet = new Set(lastUrls); const lastUrlsSet = new Set(lastUrls);
const univistedUrls = Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x)); const crawler = crawlToCrawler(
job.data.crawl_id,
sc,
sc.originUrl!,
job.data.crawlerOptions,
);
const univistedUrls = crawler.filterLinks(
Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x)),
Infinity,
sc.crawlerOptions.maxDepth ?? 10,
);
const addableJobCount = sc.crawlerOptions.limit === undefined ? Infinity : (sc.crawlerOptions.limit - await getDoneJobsOrderedLength(job.data.crawl_id)); const addableJobCount = sc.crawlerOptions.limit === undefined ? Infinity : (sc.crawlerOptions.limit - await getDoneJobsOrderedLength(job.data.crawl_id));
console.log(sc.originUrl!, univistedUrls, visitedUrls, lastUrls, addableJobCount); console.log(sc.originUrl!, univistedUrls, visitedUrls, lastUrls, addableJobCount);