AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-16 18:15:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawl-redis): batch scrape lockURL

Browse Source
This commit is contained in:
Móricz Gergő 2024-11-12 11:52:34 +01:00
parent d8bb1f68c6
commit f6db9f1428
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/lib/crawl-redis.ts
View File

@ -144,7 +144,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
await redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX");
let res: boolean;
if (!sc.crawlerOptions.deduplicateSimilarURLs) {
if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
} else {
const permutations = generateURLPermutations(url);