diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 77d392f5..2be05bd5 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -87,6 +87,7 @@ export async function runWebScraper({ crawlerOptions: crawlerOptions, pageOptions: pageOptions, priority, + teamId: team_id }); } const docs = (await provider.getDocuments(false, (progress: Progress) => { diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index e0bac57c..65247df1 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -76,6 +76,7 @@ export class WebScraperDataProvider { this.extractorOptions, existingHTML, this.priority, + this.teamId, ); processedUrls++; if (inProgress) { diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index 3dd5f9d6..67ee018c 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -80,6 +80,7 @@ export async function scrapWithFireEngine({ // atsv is only available for beta customers const betaCustomersString = process.env.BETA_CUSTOMERS; const betaCustomers = betaCustomersString ? betaCustomersString.split(",") : []; + if (pageOptions?.atsv && betaCustomers.includes(teamId)) { fireEngineOptionsParam.atsv = true; } else {