From cce94289eef2ab3c7d4034e074b01ea806b6e751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Thu, 5 Dec 2024 20:49:28 +0100
Subject: [PATCH] fix(v1/batch/scrape): horrid memory usage

---
 apps/api/src/controllers/v1/batch-scrape.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index dd8ab2fb..db029d40 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -59,6 +59,10 @@ export async function batchScrapeController(
     jobPriority = await getJobPriority({plan: req.auth.plan, team_id: req.auth.team_id, basePriority: 21})
   }
 
+  const scrapeOptions: ScrapeOptions = { ...req.body };
+  delete (scrapeOptions as any).urls;
+  delete (scrapeOptions as any).appendToId;
+
   const jobs = req.body.urls.map((x) => {
     return {
       data: {
@@ -67,7 +71,7 @@ export async function batchScrapeController(
         team_id: req.auth.team_id,
         plan: req.auth.plan!,
         crawlerOptions: null,
-        scrapeOptions: req.body,
+        scrapeOptions,
         origin: "api",
         crawl_id: id,
         sitemapped: true,