From ce2f6ff88443cdced0061463b2a0b24cbcbedb44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 15 Jan 2025 17:22:52 +0100
Subject: [PATCH] fix(queue-worker/billing): fix crawl overbilling

---
 apps/api/src/main/runWebScraper.ts    | 20 --------------------
 apps/api/src/services/queue-worker.ts | 19 ++++++++++++++++++-
 2 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 1d10fb17..c6751218 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -168,26 +168,6 @@ export async function runWebScraper({
   }
 
   if (error === undefined && response?.success) {
-    if (is_scrape === false) {
-      let creditsToBeBilled = 1; // Assuming 1 credit per document
-      if (scrapeOptions.extract) {
-        creditsToBeBilled = 5;
-      }
-
-      // If the team is the background index team, return the response
-      if (team_id === process.env.BACKGROUND_INDEX_TEAM_ID!) {
-        return response;
-      }
-
-      billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {
-        logger.error(
-          `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
-          { error },
-        );
-        // Optionally, you could notify an admin or add to a retry queue here
-      });
-    }
-
     return response;
   } else {
     if (response !== undefined) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 25ab154b..81ed955a 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -59,6 +59,7 @@ import { performExtraction } from "../lib/extract/extraction-service";
 import { supabase_service } from "../services/supabase";
 import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";
 import { saveExtract, updateExtract } from "../lib/extract/extract-redis";
+import { billTeam } from "./billing/credit_billing";
 
 configDotenv();
 
@@ -754,7 +755,6 @@ async function processJob(job: Job & { id: string }, token: string) {
     ]);
 
     if (!pipeline.success) {
-      // TODO: let's Not do this
       throw pipeline.error;
     }
 
@@ -961,6 +961,23 @@ async function processJob(job: Job & { id: string }, token: string) {
       indexJob(job, doc);
     }
 
+    if (job.data.is_scrape !== true) {
+      let creditsToBeBilled = 1; // Assuming 1 credit per document
+      if (job.data.scrapeOptions.extract) {
+        creditsToBeBilled = 5;
+      }
+
+      if (job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID!) {
+        billTeam(job.data.team_id, undefined, creditsToBeBilled, logger).catch((error) => {
+          logger.error(
+            `Failed to bill team ${job.data.team_id} for ${creditsToBeBilled} credits`,
+            { error },
+          );
+          // Optionally, you could notify an admin or add to a retry queue here
+        });
+      }
+    }
+
     logger.info(`🐂 Job done ${job.id}`);
     return data;
   } catch (error) {