AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 09:45:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(map): remove robots.txt

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-16 23:46:10 +02:00
parent 803577eeba
commit 4c1b74dab3
1 changed files with 0 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/controllers/v1/map.ts
View File

@ -28,12 +28,6 @@ export async function mapController(req: RequestWithAuth<{}, MapResponse, MapReq
const crawler = crawlToCrawler(id, sc);
try {
sc.robots = await crawler.getRobotsTxt();
} catch (e) {
Logger.debug(`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(e)}`);
}
const sitemap = sc.crawlerOptions.ignoreSitemap ? null : await crawler.tryGetSitemap();
if (sitemap !== null) {