AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 00:06:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(gcs-jobs/save): retrying saving metadata (might conflict)

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-09 09:12:27 +02:00
parent 777914e968
commit 078273695a
1 changed files with 33 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
apps/api/src/lib/gcs-jobs.ts
View File

@ -16,25 +16,39 @@ export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {
await blob.save(JSON.stringify(job.docs), {
contentType: "application/json",
});
await blob.setMetadata({
metadata: {
job_id: job.job_id ?? null,
success: job.success,
message: job.message ?? null,
num_docs: job.num_docs,
time_taken: job.time_taken,
team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_")) ? null : job.team_id,
mode: job.mode,
url: job.url,
crawler_options: job.crawlerOptions,
page_options: job.scrapeOptions,
origin: job.origin,
num_tokens: job.num_tokens ?? null,
retry: !!job.retry,
crawl_id: job.crawl_id ?? null,
tokens_billed: job.tokens_billed ?? null,
},
})
for (let i = 0; i < 3; i++) {
try {
await blob.setMetadata({
metadata: {
job_id: job.job_id ?? null,
success: job.success,
message: job.message ?? null,
num_docs: job.num_docs,
time_taken: job.time_taken,
team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_")) ? null : job.team_id,
mode: job.mode,
url: job.url,
crawler_options: job.crawlerOptions,
page_options: job.scrapeOptions,
origin: job.origin,
num_tokens: job.num_tokens ?? null,
retry: !!job.retry,
crawl_id: job.crawl_id ?? null,
tokens_billed: job.tokens_billed ?? null,
},
});
break;
} catch (error) {
logger.error(`Error saving job metadata to GCS`, {
error,
scrapeId: job.job_id,
jobId: job.job_id,
});
if (i === 2) {
throw error;
}
}
}
} catch (error) {
logger.error(`Error saving job to GCS`, {
error,