diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 4517c7b6..a8039d68 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -310,6 +310,7 @@ const baseScrapeOptions = z blockAds: z.boolean().default(true), proxy: z.enum(["basic", "stealth", "auto"]).optional(), maxAge: z.number().int().gte(0).safe().default(0), + dontStoreInCache: z.boolean().default(false), }) .strict(strictMessage); diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 5caee212..26b24e31 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -6,7 +6,8 @@ import { EngineError, IndexMissError } from "../../error"; import crypto from "crypto"; export async function sendDocumentToIndex(meta: Meta, document: Document) { - const shouldCache = meta.winnerEngine !== "cache" + const shouldCache = !meta.options.dontStoreInCache + && meta.winnerEngine !== "cache" && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents" && !meta.featureFlags.has("actions")