AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 15:39:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update log_job.ts (#1263)

Browse Source
This commit is contained in:
Nicolas 2025-02-25 19:35:57 -03:00 committed by GitHub
parent 115b6b61c4
commit 0f0520391a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 42 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
apps/api/src/services/logging/log_job.ts
View File

@ -21,6 +21,43 @@ function cleanOfNull<T>(x: T): T {
}
}
async function indexJob(job: FirecrawlJob): Promise<void> {
try {
const response = await fetch(`${process.env.FIRE_INDEX_SERVER_URL}/api/jobs`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
url: job.url,
mode: job.mode || "scrape",
docs: job.docs,
origin: job.origin,
success: job.success,
time_taken: job.time_taken,
num_tokens: job.num_tokens,
page_options: job.scrapeOptions,
date_added: new Date().toISOString(),
}),
});
if (!response.ok) {
const errorData = await response.json();
logger.error(`Failed to send job to external server: ${response.status} ${response.statusText}`, {
error: errorData,
scrapeId: job.job_id,
});
} else {
logger.debug("Job sent to external server successfully!", { scrapeId: job.job_id });
}
} catch (error) {
logger.error(`Error sending job to external server: ${error.message}`, {
error,
scrapeId: job.job_id,
});
}
}
export async function logJob(job: FirecrawlJob, force: boolean = false) {
try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
@ -62,6 +99,11 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {
tokens_billed: job.tokens_billed,
};
// Send job to external server
if (process.env.FIRE_INDEX_SERVER_URL) {
indexJob(job);
}
if (force) {
let i = 0,
done = false;