From 59d09f5c45c3f27bf4d07148b60c5c4d8b952c1e Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 25 Feb 2025 19:32:16 -0300 Subject: [PATCH 1/3] Update log_job.ts --- apps/api/src/services/logging/log_job.ts | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index 707f2911..5639c9bf 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -21,6 +21,43 @@ function cleanOfNull(x: T): T { } } +async function indexJob(job: FirecrawlJob): Promise { + try { + const response = await fetch(`${process.env.FIRE_INDEX_SERVER_URL}/api/jobs`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + url: job.url, + mode: job.mode || "scrape", + docs: job.docs, + origin: job.origin, + success: job.success, + time_taken: job.time_taken, + num_tokens: job.num_tokens, + page_options: job.scrapeOptions, + date_added: new Date().toISOString(), + }), + }); + + if (!response.ok) { + const errorData = await response.json(); + logger.error(`Failed to send job to external server: ${response.status} ${response.statusText}`, { + error: errorData, + scrapeId: job.job_id, + }); + } else { + logger.debug("Job sent to external server successfully!", { scrapeId: job.job_id }); + } + } catch (error) { + logger.error(`Error sending job to external server: ${error.message}`, { + error, + scrapeId: job.job_id, + }); + } +} + export async function logJob(job: FirecrawlJob, force: boolean = false) { try { const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true"; @@ -62,6 +99,11 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) { tokens_billed: job.tokens_billed, }; + // Send job to external server + if (process.env.FIRE_INDEX_SERVER_URL) { + indexJob(job); + } + if (force) { let i = 0, done = false; From ec90aaffd6ab539377e2929984160c4659b15d06 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 25 Feb 2025 21:01:00 -0300 Subject: [PATCH 2/3] Update log_job.ts --- apps/api/src/services/logging/log_job.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index 5639c9bf..29c748c6 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -23,6 +23,11 @@ function cleanOfNull(x: T): T { async function indexJob(job: FirecrawlJob): Promise { try { + // Skip indexing for anything except single_urls and scrape modes + if (job.mode !== "single_urls" && job.mode !== "scrape") { + return; + } + const response = await fetch(`${process.env.FIRE_INDEX_SERVER_URL}/api/jobs`, { method: 'POST', headers: { From 31df234127356b3c9617b841f49488fb88d130e9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 25 Feb 2025 21:01:05 -0300 Subject: [PATCH 3/3] Update log_job.ts --- apps/api/src/services/logging/log_job.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index 29c748c6..ac41b6e8 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -23,7 +23,6 @@ function cleanOfNull(x: T): T { async function indexJob(job: FirecrawlJob): Promise { try { - // Skip indexing for anything except single_urls and scrape modes if (job.mode !== "single_urls" && job.mode !== "scrape") { return; }