From 4709fbce6aa868253b0697c2afbd412f3f44a4da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 9 Apr 2025 09:29:37 +0200
Subject: [PATCH] feat(gcs-jobs/save): retry save operation

---
 apps/api/src/lib/gcs-jobs.ts | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 983d4771..2f58a343 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -13,9 +13,25 @@ export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {
         const storage = new Storage({ credentials });
         const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
         const blob = bucket.file(`${job.job_id}.json`);
-        await blob.save(JSON.stringify(job.docs), {
-            contentType: "application/json",
-        });
+        for (let i = 0; i < 3; i++) {
+            try {
+                await blob.save(JSON.stringify(job.docs), { 
+                    contentType: "application/json",
+                });
+                break;
+            } catch (error) {
+                if (i === 2) {
+                    throw error;
+                } else {
+                    logger.error(`Error saving job to GCS, retrying`, {
+                        error,
+                        scrapeId: job.job_id,
+                        jobId: job.job_id,
+                        i,
+                    });
+                }
+            }
+        }
         for (let i = 0; i < 3; i++) {
             try {
                 await blob.setMetadata({
@@ -39,13 +55,15 @@ export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {
                 });
                 break;
             } catch (error) {
-                logger.error(`Error saving job metadata to GCS`, {
-                    error,
-                    scrapeId: job.job_id,
-                    jobId: job.job_id,
-                });
                 if (i === 2) {
                     throw error;
+                } else {
+                    logger.error(`Error saving job metadata to GCS, retrying`, {
+                        error,
+                        scrapeId: job.job_id,
+                        jobId: job.job_id,
+                        i,
+                    });
                 }
             }
         }