From ba2af74adf6f174c5bb7bc8fe32fd128e7c16e6e Mon Sep 17 00:00:00 2001 From: Kevin Swiber Date: Fri, 9 Aug 2024 15:29:18 -0700 Subject: [PATCH] Ensuring USE_DB_AUTHENTICATION is true in single URL scraper. --- apps/api/src/scraper/WebScraper/single_url.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 12e075fd..96adcf49 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -23,11 +23,13 @@ import { clientSideError } from "../../strings"; dotenv.config(); +const useDatabaseAuth = process.env.USE_DB_AUTHENTICATION === "true"; + export const baseScrapers = [ "fire-engine", "fire-engine;chrome-cdp", "scrapingBee", - process.env.USE_DB_AUTHENTICATION ? undefined : "playwright", + useDatabaseAuth ? undefined : "playwright", "scrapingBeeLoad", "fetch", ].filter(Boolean); @@ -85,10 +87,10 @@ function getScrapingFallbackOrder( }); let defaultOrder = [ - !process.env.USE_DB_AUTHENTICATION ? undefined : "fire-engine", - !process.env.USE_DB_AUTHENTICATION ? undefined : "fire-engine;chrome-cdp", + !useDatabaseAuth ? undefined : "fire-engine", + !useDatabaseAuth ? undefined : "fire-engine;chrome-cdp", "scrapingBee", - process.env.USE_DB_AUTHENTICATION ? undefined : "playwright", + useDatabaseAuth ? undefined : "playwright", "scrapingBeeLoad", "fetch", ].filter(Boolean); @@ -96,7 +98,7 @@ function getScrapingFallbackOrder( if (isWaitPresent || isScreenshotPresent || isHeadersPresent) { defaultOrder = [ "fire-engine", - process.env.USE_DB_AUTHENTICATION ? undefined : "playwright", + useDatabaseAuth ? undefined : "playwright", ...defaultOrder.filter( (scraper) => scraper !== "fire-engine" && scraper !== "playwright" ),