AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 12:25:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(v1/batch/scrape): horrid memory usage

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-05 20:49:28 +01:00
parent f8e619b5df
commit cce94289ee
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/controllers/v1/batch-scrape.ts
View File

@ -59,6 +59,10 @@ export async function batchScrapeController(
jobPriority = await getJobPriority({plan: req.auth.plan, team_id: req.auth.team_id, basePriority: 21}) jobPriority = await getJobPriority({plan: req.auth.plan, team_id: req.auth.team_id, basePriority: 21})
} }
const scrapeOptions: ScrapeOptions = { ...req.body };
delete (scrapeOptions as any).urls;
delete (scrapeOptions as any).appendToId;
const jobs = req.body.urls.map((x) => { const jobs = req.body.urls.map((x) => {
return { return {
data: { data: {
@ -67,7 +71,7 @@ export async function batchScrapeController(
team_id: req.auth.team_id, team_id: req.auth.team_id,
plan: req.auth.plan!, plan: req.auth.plan!,
crawlerOptions: null, crawlerOptions: null,
scrapeOptions: req.body, scrapeOptions,
origin: "api", origin: "api",
crawl_id: id, crawl_id: id,
sitemapped: true, sitemapped: true,