From 5b0f2da02f4e6d082c50f66f5d9ebe5de85b1734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 29 May 2025 15:23:24 +0200 Subject: [PATCH] feat: dontStoreInCache --- apps/api/src/controllers/v1/types.ts | 1 + apps/api/src/scraper/scrapeURL/engines/index/index.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 4517c7b6..a8039d68 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -310,6 +310,7 @@ const baseScrapeOptions = z blockAds: z.boolean().default(true), proxy: z.enum(["basic", "stealth", "auto"]).optional(), maxAge: z.number().int().gte(0).safe().default(0), + dontStoreInCache: z.boolean().default(false), }) .strict(strictMessage); diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 5caee212..26b24e31 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -6,7 +6,8 @@ import { EngineError, IndexMissError } from "../../error"; import crypto from "crypto"; export async function sendDocumentToIndex(meta: Meta, document: Document) { - const shouldCache = meta.winnerEngine !== "cache" + const shouldCache = !meta.options.dontStoreInCache + && meta.winnerEngine !== "cache" && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents" && !meta.featureFlags.has("actions")