diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts index 982b63c4..afa7fc9e 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index.ts @@ -439,9 +439,18 @@ export function buildFallbackList(meta: Meta): { ) && ( meta.options.actions === undefined - || Object.keys(meta.options.actions).length === 0 + || meta.options.actions.length === 0 ) && meta.options.proxy !== "stealth"; + + meta.logger.warn("shouldUseIndex", { + shouldUseIndex, + formatsNoChangeTracking: !meta.options.formats.includes("changeTracking"), + maxAge: meta.options.maxAge !== 0, + headers: meta.options.headers === undefined || Object.keys(meta.options.headers).length === 0, + actions: meta.options.actions === undefined || meta.options.actions.length === 0, + proxy: meta.options.proxy !== "stealth", + }); if (!shouldUseIndex) { const indexIndex = _engines.indexOf("index"); diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 15daf1f9..cc96e399 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -15,6 +15,13 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { || Object.keys(meta.options.headers).length === 0 ); + meta.logger.warn("shouldCache", { + shouldCache, + winnerEngine: meta.winnerEngine !== "cache" && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents", + featureFlags: !meta.featureFlags.has("actions"), + headers: meta.options.headers === undefined || Object.keys(meta.options.headers).length === 0, + }); + if (!shouldCache) { return document; } @@ -57,8 +64,8 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { has_screenshot_fullscreen: document.screenshot !== undefined && meta.featureFlags.has("screenshot@fullScreen"), is_mobile: meta.options.mobile, block_ads: meta.options.blockAds, - location: meta.options.location?.country ?? null, - languages: meta.options.location?.languages ?? null, + location_country: meta.options.location?.country ?? null, + location_languages: meta.options.location?.languages ?? null, status: document.metadata.statusCode, });