diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index c163aa60..d2977cea 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -331,7 +331,7 @@ describe("E2E Tests for API Routes", () => { expect(completedResponse.body.data[0].content).toContain("Mendable"); expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200); expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined(); - }, 60000); // 60 seconds + }, 180000); // 180 seconds it.concurrent("should return a successful response with a valid API key and valid excludes option", async () => { const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL) diff --git a/apps/api/src/controllers/crawl-status.ts b/apps/api/src/controllers/crawl-status.ts index 448c119c..3a079455 100644 --- a/apps/api/src/controllers/crawl-status.ts +++ b/apps/api/src/controllers/crawl-status.ts @@ -40,7 +40,7 @@ export async function crawlStatusController(req: Request, res: Response) { current_url, current_step, total, - data, + data: data ? data : null, partial_data: jobStatus == 'completed' ? [] : partialDocs, }); } catch (error) { diff --git a/apps/api/src/controllers/status.ts b/apps/api/src/controllers/status.ts index 90797874..756c4241 100644 --- a/apps/api/src/controllers/status.ts +++ b/apps/api/src/controllers/status.ts @@ -1,5 +1,6 @@ import { Request, Response } from "express"; import { getWebScraperQueue } from "../../src/services/queue-service"; +import { supabaseGetJobById } from "../../src/lib/supabase-jobs"; export async function crawlJobStatusPreviewController(req: Request, res: Response) { try { @@ -9,15 +10,26 @@ export async function crawlJobStatusPreviewController(req: Request, res: Respons } const { current, current_url, total, current_step, partialDocs } = await job.progress(); + let data = job.returnvalue; + if (process.env.USE_DB_AUTHENTICATION) { + const supabaseData = await supabaseGetJobById(req.params.jobId); + + if (supabaseData) { + data = supabaseData.docs; + } + } + + const jobStatus = await job.getState(); + res.json({ - status: await job.getState(), + status: jobStatus, // progress: job.progress(), - current: current, - current_url: current_url, - current_step: current_step, - total: total, - data: job.returnvalue, - partial_data: partialDocs ?? [], + current, + current_url, + current_step, + total, + data: data ? data : null, + partial_data: jobStatus == 'completed' ? [] : partialDocs, }); } catch (error) { console.error(error); diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts index 4dab0aea..1f9531e5 100644 --- a/apps/api/src/lib/supabase-jobs.ts +++ b/apps/api/src/lib/supabase-jobs.ts @@ -8,7 +8,6 @@ export const supabaseGetJobById = async (jobId: string) => { .single(); if (error) { - console.error('Error while fetching supabase for job:', jobId, 'error:', error); return null; } diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 57d6f322..9f35c4d0 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -110,15 +110,20 @@ export async function runWebScraper({ } const saveJob = async (job: Job, result: any) => { - if (process.env.USE_DB_AUTHENTICATION) { - const { data, error } = await supabase_service - .from("firecrawl_jobs") - .update({ docs: result }) - .eq("job_id", job.id); + try { + if (process.env.USE_DB_AUTHENTICATION) { + const { data, error } = await supabase_service + .from("firecrawl_jobs") + .update({ docs: result }) + .eq("job_id", job.id); - job.moveToCompleted(null); // returnvalue - } else { - job.moveToCompleted(result); // returnvalue + if (error) throw new Error(error.message); + await job.moveToCompleted(null); + } else { + await job.moveToCompleted(result); + } + } catch (error) { + console.error("Failed to update job status:", error); } } diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index d4810770..439f900e 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -6,10 +6,9 @@ import "dotenv/config"; export async function logJob(job: FirecrawlJob) { try { - // Only log jobs in production - // if (process.env.ENV !== "production") { - // return; - // } + if (!process.env.USE_DB_AUTHENTICATION) { + return; + } const { data, error } = await supabase_service .from("firecrawl_jobs")