From 25dd16bf2a6d534864623062526e49e405d9d26f Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 3 Oct 2024 18:52:17 -0300 Subject: [PATCH] Nick: removed 401 --- apps/api/src/scraper/WebScraper/single_url.ts | 2 +- apps/redis/start-redis-server.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 767f30e1..13ca7dd2 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -425,7 +425,7 @@ export async function scrapSingleUrl( Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`); break; } - if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400 || pageStatusCode == 401)) { + if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) { Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`); break; } diff --git a/apps/redis/start-redis-server.sh b/apps/redis/start-redis-server.sh index ed252fde..30e4a145 100755 --- a/apps/redis/start-redis-server.sh +++ b/apps/redis/start-redis-server.sh @@ -28,3 +28,4 @@ redis-server $PW_ARG \ --maxmemory-policy $MAXMEMORY_POLICY \ --appendonly $APPENDONLY \ --save "$SAVE" + --save "$SAVE" \