looking good

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -331,7 +331,7 @@ describe("E2E Tests for API Routes", () => {
       expect(completedResponse.body.data[0].content).toContain("Mendable");
       expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
       expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
-    }, 60000); // 60 seconds
+    }, 180000); // 180 seconds
 
     it.concurrent("should return a successful response with a valid API key and valid excludes option", async () => {
       const crawlResponse: FirecrawlCrawlResponse = await request(TEST_URL)

apps/api/src/controllers/crawl-status.ts

@@ -40,7 +40,7 @@ export async function crawlStatusController(req: Request, res: Response) {
       current_url,
       current_step,
       total,
-      data,
+      data: data ? data : null,
       partial_data: jobStatus == 'completed' ? [] : partialDocs,
     });
   } catch (error) {

apps/api/src/controllers/status.ts

@@ -1,5 +1,6 @@
 import { Request, Response } from "express";
 import { getWebScraperQueue } from "../../src/services/queue-service";
+import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
 
 export async function crawlJobStatusPreviewController(req: Request, res: Response) {
   try {
@@ -9,15 +10,26 @@ export async function crawlJobStatusPreviewController(req: Request, res: Respons
     }
 
     const { current, current_url, total, current_step, partialDocs } = await job.progress();
+    let data = job.returnvalue;
+    if (process.env.USE_DB_AUTHENTICATION) {
+      const supabaseData = await supabaseGetJobById(req.params.jobId);
+
+      if (supabaseData) {
+        data = supabaseData.docs;
+      }
+    }
+
+    const jobStatus = await job.getState();
+
     res.json({
-      status: await job.getState(),
+      status: jobStatus,
       // progress: job.progress(),
-      current: current,
+      current,
-      current_url: current_url,
+      current_url,
-      current_step: current_step,
+      current_step,
-      total: total,
+      total,
-      data: job.returnvalue,
+      data: data ? data : null,
-      partial_data: partialDocs ?? [],
+      partial_data: jobStatus == 'completed' ? [] : partialDocs,
     });
   } catch (error) {
     console.error(error);

apps/api/src/lib/supabase-jobs.ts

@@ -8,7 +8,6 @@ export const supabaseGetJobById = async (jobId: string) => {
     .single();
 
   if (error) {
-    console.error('Error while fetching supabase for job:', jobId, 'error:', error);
     return null;
   }
 

apps/api/src/main/runWebScraper.ts

@@ -110,15 +110,20 @@ export async function runWebScraper({
 }
 
 const saveJob = async (job: Job, result: any) => {
-  if (process.env.USE_DB_AUTHENTICATION) {
+  try {
-    const { data, error } = await supabase_service
+    if (process.env.USE_DB_AUTHENTICATION) {
-      .from("firecrawl_jobs")
+      const { data, error } = await supabase_service
-      .update({ docs: result })
+        .from("firecrawl_jobs")
-      .eq("job_id", job.id);
+        .update({ docs: result })
+        .eq("job_id", job.id);
 
-    job.moveToCompleted(null); // returnvalue
+      if (error) throw new Error(error.message);
-  } else {
+      await job.moveToCompleted(null);
-    job.moveToCompleted(result); // returnvalue
+    } else {
+      await job.moveToCompleted(result);
+    }
+  } catch (error) {
+    console.error("Failed to update job status:", error);
   }
 }
 

apps/api/src/services/logging/log_job.ts

@@ -6,10 +6,9 @@ import "dotenv/config";
 
 export async function logJob(job: FirecrawlJob) {
   try {
-    // Only log jobs in production
+    if (!process.env.USE_DB_AUTHENTICATION) {
-    // if (process.env.ENV !== "production") {
+      return;
-    //   return;
+    }
-    // }
 
     const { data, error } = await supabase_service
       .from("firecrawl_jobs")