From c5597bc7221f2d0d0504e322ddf4e7e9c219f900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Thu, 15 Aug 2024 19:11:07 +0200
Subject: [PATCH] fix: robots.txt laoding

---
 apps/api/src/controllers/crawl.ts | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/apps/api/src/controllers/crawl.ts b/apps/api/src/controllers/crawl.ts
index 4f3c5f22..5c1e0428 100644
--- a/apps/api/src/controllers/crawl.ts
+++ b/apps/api/src/controllers/crawl.ts
@@ -95,25 +95,22 @@ export async function crawlController(req: Request, res: Response) {
 
     await logCrawl(id, team_id);
 
-    let robots;
-
-    try {
-      robots = await this.getRobotsTxt();
-    } catch (_) {}
-
     const sc: StoredCrawl = {
       originUrl: url,
       crawlerOptions,
       pageOptions,
       team_id,
-      robots,
       createdAt: Date.now(),
     };
 
-    await saveCrawl(id, sc);
-
     const crawler = crawlToCrawler(id, sc);
 
+    try {
+      sc.robots = await crawler.getRobotsTxt();
+    } catch (_) {}
+
+    await saveCrawl(id, sc);
+
     const sitemap = sc.crawlerOptions?.ignoreSitemap ? null : await crawler.tryGetSitemap();
 
     if (sitemap !== null) {