AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-08 16:49:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawl-redis/lockURL): only add to visited_unique if lock succeeds

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-15 21:01:31 +01:00
parent 30fa78cd9e
commit 37f58efe45
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
apps/api/src/lib/crawl-redis.ts
View File

@ -233,13 +233,6 @@ export async function lockURL(
url = normalizeURL(url, sc);
logger = logger.child({ url });
await redisConnection.sadd("crawl:" + id + ":visited_unique", url);
await redisConnection.expire(
"crawl:" + id + ":visited_unique",
24 * 60 * 60,
"NX",
);
let res: boolean;
if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0;
@ -255,6 +248,15 @@ export async function lockURL(
await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");
if (res) {
await redisConnection.sadd("crawl:" + id + ":visited_unique", url);
await redisConnection.expire(
"crawl:" + id + ":visited_unique",
24 * 60 * 60,
"NX",
);
}
logger.debug("Locking URL " + JSON.stringify(url) + "... result: " + res, {
res,
});