AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-21 15:27:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawl-redis/lockURL): reduce logging

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-09 19:29:42 +01:00
parent 6776aee1c3
commit 91a1a9a1fc
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/lib/crawl-redis.ts
View File

@ -151,7 +151,6 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
url = normalizeURL(url, sc);
logger = logger.child({ url });
logger.debug("Locking URL " + JSON.stringify(url) + "...");
await redisConnection.sadd("crawl:" + id + ":visited_unique", url);
await redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX");
@ -160,14 +159,14 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
} else {
const permutations = generateURLPermutations(url).map(x => x.href);
logger.debug("Adding URL permutations for URL " + JSON.stringify(url) + "...", { permutations });
// logger.debug("Adding URL permutations for URL " + JSON.stringify(url) + "...", { permutations });
const x = (await redisConnection.sadd("crawl:" + id + ":visited", ...permutations));
res = x === permutations.length;
}
await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");
logger.debug("lockURL final result: " + res, { res });
logger.debug("Locking URL " + JSON.stringify(url) + "... result: " + res, { res });
return res;
}