diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts index cb793572..0df700f4 100644 --- a/apps/api/src/lib/extract/extract-redis.ts +++ b/apps/api/src/lib/extract/extract-redis.ts @@ -54,6 +54,19 @@ export async function updateExtract( extract.steps = [...current.steps, ...extract.steps]; } + // Limit links in steps to 500 + if (extract.steps) { + extract.steps = extract.steps.map(step => { + if (step.discoveredLinks && step.discoveredLinks.length > 500) { + return { + ...step, + discoveredLinks: step.discoveredLinks.slice(0, 500) + }; + } + return step; + }); + } + await redisConnection.set( "extract:" + id, JSON.stringify({ ...current, ...extract }),