AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 15:59:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawler): dumb sitemap limit

Browse Source
This commit is contained in:
Móricz Gergő 2025-01-23 07:10:07 +01:00
parent 51a0e233e3
commit a922aac805
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -27,6 +27,7 @@ export class WebCrawler {
private allowSubdomains: boolean; private allowSubdomains: boolean;
private ignoreRobotsTxt: boolean; private ignoreRobotsTxt: boolean;
private logger: typeof _logger; private logger: typeof _logger;
private sitemapsHit: Set<string> = new Set();
constructor({ constructor({
jobId, jobId,
@ -531,10 +532,22 @@ export class WebCrawler {
url: string, url: string,
urlsHandler: (urls: string[]) => unknown, urlsHandler: (urls: string[]) => unknown,
): Promise<number> { ): Promise<number> {
if (this.sitemapsHit.size >= 5) {
this.logger.warn("Sitemap limit of 5 hit, not hitting this one.");
return 0;
}
const sitemapUrl = url.endsWith(".xml") const sitemapUrl = url.endsWith(".xml")
? url ? url
: `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`; : `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;
if (this.sitemapsHit.has(sitemapUrl)) {
this.logger.warn("This sitemap has already been hit.", { sitemapUrl });
return 0;
}
this.sitemapsHit.add(sitemapUrl);
let sitemapCount: number = 0; let sitemapCount: number = 0;
// Try to get sitemap from the provided URL first // Try to get sitemap from the provided URL first