From 4c1b74dab33782320bf1392255c105638b4493fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Fri, 16 Aug 2024 23:46:10 +0200 Subject: [PATCH] fix(map): remove robots.txt --- apps/api/src/controllers/v1/map.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index 5dbe4b66..71579b38 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -28,12 +28,6 @@ export async function mapController(req: RequestWithAuth<{}, MapResponse, MapReq const crawler = crawlToCrawler(id, sc); - try { - sc.robots = await crawler.getRobotsTxt(); - } catch (e) { - Logger.debug(`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(e)}`); - } - const sitemap = sc.crawlerOptions.ignoreSitemap ? null : await crawler.tryGetSitemap(); if (sitemap !== null) {