From c323c64671736a346e5bf3631cefacc306809a60 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 14 Jan 2025 02:00:47 -0300 Subject: [PATCH] Update extract-redis.ts --- apps/api/src/lib/extract/extract-redis.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts index cb793572..0df700f4 100644 --- a/apps/api/src/lib/extract/extract-redis.ts +++ b/apps/api/src/lib/extract/extract-redis.ts @@ -54,6 +54,19 @@ export async function updateExtract( extract.steps = [...current.steps, ...extract.steps]; } + // Limit links in steps to 500 + if (extract.steps) { + extract.steps = extract.steps.map(step => { + if (step.discoveredLinks && step.discoveredLinks.length > 500) { + return { + ...step, + discoveredLinks: step.discoveredLinks.slice(0, 500) + }; + } + return step; + }); + } + await redisConnection.set( "extract:" + id, JSON.stringify({ ...current, ...extract }),