AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 05:38:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(gcs-jobs/save): retry save operation

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-09 09:29:37 +02:00
parent 078273695a
commit 4709fbce6a
1 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
apps/api/src/lib/gcs-jobs.ts
View File

@ -13,9 +13,25 @@ export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {
const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
const blob = bucket.file(`${job.job_id}.json`);
await blob.save(JSON.stringify(job.docs), {
contentType: "application/json",
});
for (let i = 0; i < 3; i++) {
try {
await blob.save(JSON.stringify(job.docs), {
contentType: "application/json",
});
break;
} catch (error) {
if (i === 2) {
throw error;
} else {
logger.error(`Error saving job to GCS, retrying`, {
error,
scrapeId: job.job_id,
jobId: job.job_id,
i,
});
}
}
}
for (let i = 0; i < 3; i++) {
try {
await blob.setMetadata({
@ -39,13 +55,15 @@ export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {
});
break;
} catch (error) {
logger.error(`Error saving job metadata to GCS`, {
error,
scrapeId: job.job_id,
jobId: job.job_id,
});
if (i === 2) {
throw error;
} else {
logger.error(`Error saving job metadata to GCS, retrying`, {
error,
scrapeId: job.job_id,
jobId: job.job_id,
i,
});
}
}
}