AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 21:29:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

crawl fixes

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-20 19:47:58 +01:00
parent 103c3f28e6
commit b468bb4014
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/lib/crawl-redis.ts
View File

@ -148,7 +148,8 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
} else {
const permutations = generateURLPermutations(url);
res = (await redisConnection.sadd("crawl:" + id + ":visited", ...permutations.map(x => x.href))) === permutations.length;
const x = (await redisConnection.sadd("crawl:" + id + ":visited", ...permutations.map(x => x.href)));
res = x === permutations.length;
}
await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");

2
apps/api/src/services/queue-worker.ts
View File

@ -350,7 +350,7 @@ async function processJob(job: Job & { id: string }, token: string) {
await addCrawlJobDone(job.data.crawl_id, job.id);
if (!job.data.sitemapped && job.data.crawlerOptions !== null) {
if (job.data.crawlerOptions !== null) {
if (!sc.cancelled) {
const crawler = crawlToCrawler(job.data.crawl_id, sc, doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl);