AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-08 20:40:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawl-redis): oops

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-05 21:42:08 +01:00
parent 845c2744a9
commit f82b9c205c
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/lib/crawl-redis.ts
View File

@ -139,7 +139,7 @@ export function generateURLPermutations(url: string | URL): URL[] {
}
export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise<boolean> {
const logger = _logger.child({ crawlId: id, module: "crawl-redis", method: "lockURL", preNormalizedURL: url, teamId: sc.team_id, plan: sc.plan });
let logger = _logger.child({ crawlId: id, module: "crawl-redis", method: "lockURL", preNormalizedURL: url, teamId: sc.team_id, plan: sc.plan });
if (typeof sc.crawlerOptions?.limit === "number") {
if (await redisConnection.scard("crawl:" + id + ":visited_unique") >= sc.crawlerOptions.limit) {
@ -149,7 +149,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
}
url = normalizeURL(url, sc);
logger.defaultMeta.url = url;
logger = logger.child({ url });
logger.debug("Locking URL " + JSON.stringify(url) + "...");
await redisConnection.sadd("crawl:" + id + ":visited_unique", url);