AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 01:36:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'nsc/__experimental_streamSteps'

Browse Source
This commit is contained in:
Nicolas 2025-01-14 02:00:58 -03:00
commit 23d3257a57
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/lib/extract/extract-redis.ts
View File

@ -54,6 +54,19 @@ export async function updateExtract(
extract.steps = [...current.steps, ...extract.steps]; extract.steps = [...current.steps, ...extract.steps];
} }
// Limit links in steps to 500
if (extract.steps) {
extract.steps = extract.steps.map(step => {
if (step.discoveredLinks && step.discoveredLinks.length > 500) {
return {
...step,
discoveredLinks: step.discoveredLinks.slice(0, 500)
};
}
return step;
});
}
await redisConnection.set( await redisConnection.set(
"extract:" + id, "extract:" + id,
JSON.stringify({ ...current, ...extract }), JSON.stringify({ ...current, ...extract }),

4
apps/api/src/lib/extract/extraction-service.ts
View File

@ -256,7 +256,7 @@ export async function performExtraction(
step: ExtractStep.MULTI_ENTITY_SCRAPE, step: ExtractStep.MULTI_ENTITY_SCRAPE,
startedAt: Date.now(), startedAt: Date.now(),
finishedAt: Date.now(), finishedAt: Date.now(),
discoveredLinks: [], discoveredLinks: links,
}, },
], ],
}); });
@ -510,7 +510,7 @@ export async function performExtraction(
step: ExtractStep.EXTRACT, step: ExtractStep.EXTRACT,
startedAt: Date.now(), startedAt: Date.now(),
finishedAt: Date.now(), finishedAt: Date.now(),
discoveredLinks: [], discoveredLinks: links,
}, },
], ],
}); });