AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 16:20:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(log_job): add force option to retry on supabase failure

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-15 19:55:23 +01:00
parent 7b02c45dd0
commit ca2e33db0a
2 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
apps/api/src/services/logging/log_job.ts
View File

@ -7,7 +7,7 @@ import { logger } from "../../lib/logger";
import { configDotenv } from "dotenv"; import { configDotenv } from "dotenv";
configDotenv(); configDotenv();
export async function logJob(job: FirecrawlJob) { export async function logJob(job: FirecrawlJob, force: boolean = false) {
try { try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
if (!useDbAuthentication) { if (!useDbAuthentication) {
@ -23,11 +23,7 @@ export async function logJob(job: FirecrawlJob) {
job.scrapeOptions.headers["Authorization"] = "REDACTED"; job.scrapeOptions.headers["Authorization"] = "REDACTED";
job.docs = [{ content: "REDACTED DUE TO AUTHORIZATION HEADER", html: "REDACTED DUE TO AUTHORIZATION HEADER" }]; job.docs = [{ content: "REDACTED DUE TO AUTHORIZATION HEADER", html: "REDACTED DUE TO AUTHORIZATION HEADER" }];
} }
const jobColumn = {
const { data, error } = await supabase_service
.from("firecrawl_jobs")
.insert([
{
job_id: job.job_id ? job.job_id : null, job_id: job.job_id ? job.job_id : null,
success: job.success, success: job.success,
message: job.message, message: job.message,
@ -43,8 +39,36 @@ export async function logJob(job: FirecrawlJob) {
num_tokens: job.num_tokens, num_tokens: job.num_tokens,
retry: !!job.retry, retry: !!job.retry,
crawl_id: job.crawl_id, crawl_id: job.crawl_id,
}, };
]);
if (force) {
while (true) {
try {
const { error } = await supabase_service
.from("firecrawl_jobs")
.insert([jobColumn]);
if (error) {
logger.error("Failed to log job due to Supabase error -- trying again", { error, scrapeId: job.job_id });
await new Promise<void>((resolve) => setTimeout(() => resolve(), 75));
} else {
break;
}
} catch (error) {
logger.error("Failed to log job due to thrown error -- trying again", { error, scrapeId: job.job_id });
await new Promise<void>((resolve) => setTimeout(() => resolve(), 75));
}
}
logger.debug("Job logged successfully!", { scrapeId: job.job_id });
} else {
const { error } = await supabase_service
.from("firecrawl_jobs")
.insert([jobColumn]);
if (error) {
logger.error(`Error logging job: ${error.message}`, { error, scrapeId: job.job_id });
} else {
logger.debug("Job logged successfully!", { scrapeId: job.job_id });
}
}
if (process.env.POSTHOG_API_KEY && !job.crawl_id) { if (process.env.POSTHOG_API_KEY && !job.crawl_id) {
let phLog = { let phLog = {
@ -72,9 +96,7 @@ export async function logJob(job: FirecrawlJob) {
posthog.capture(phLog); posthog.capture(phLog);
} }
} }
if (error) {
logger.error(`Error logging job: ${error.message}`);
}
} catch (error) { } catch (error) {
logger.error(`Error logging job: ${error.message}`); logger.error(`Error logging job: ${error.message}`);
} }

6
apps/api/src/services/queue-worker.ts
View File

@ -346,7 +346,7 @@ async function processJob(job: Job & { id: string }, token: string) {
scrapeOptions: job.data.scrapeOptions, scrapeOptions: job.data.scrapeOptions,
origin: job.data.origin, origin: job.data.origin,
crawl_id: job.data.crawl_id, crawl_id: job.data.crawl_id,
}); }, true);
await addCrawlJobDone(job.data.crawl_id, job.id); await addCrawlJobDone(job.data.crawl_id, job.id);
@ -486,7 +486,7 @@ async function processJob(job: Job & { id: string }, token: string) {
url: sc?.originUrl ?? (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"), url: sc?.originUrl ?? (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
crawlerOptions: sc.crawlerOptions, crawlerOptions: sc.crawlerOptions,
origin: job.data.origin, origin: job.data.origin,
}); }, true);
} }
} }
} }
@ -566,7 +566,7 @@ async function processJob(job: Job & { id: string }, token: string) {
scrapeOptions: job.data.scrapeOptions, scrapeOptions: job.data.scrapeOptions,
origin: job.data.origin, origin: job.data.origin,
crawl_id: job.data.crawl_id, crawl_id: job.data.crawl_id,
}); }, true);
// await logJob({ // await logJob({
// job_id: job.data.crawl_id, // job_id: job.data.crawl_id,