feat: improve billing logging

apps/api/src/controllers/v0/scrape.ts

@@ -265,9 +265,10 @@ export async function scrapeController(req: Request, res: Response) {
       }
       if (creditsToBeBilled > 0) {
         // billing for doc done on queue end, bill only for llm extraction
-        billTeam(team_id, chunk?.sub_id, creditsToBeBilled).catch((error) => {
+        billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch((error) => {
           logger.error(
-            `Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`,
+            `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
+            { error }
           );
           // Optionally, you could notify an admin or add to a retry queue here
         });
@@ -312,7 +313,7 @@ export async function scrapeController(req: Request, res: Response) {
     return res.status(result.returnCode).json(result);
   } catch (error) {
     Sentry.captureException(error);
-    logger.error(error);
+    logger.error("Scrape error occcurred", { error });
     return res.status(500).json({
       error:
         error instanceof ZodError

apps/api/src/main/runWebScraper.ts

@@ -7,7 +7,7 @@ import {
 import { billTeam } from "../services/billing/credit_billing";
 import { Document } from "../controllers/v1/types";
 import { supabase_service } from "../services/supabase";
-import { logger } from "../lib/logger";
+import { logger as _logger } from "../lib/logger";
 import { ScrapeEvents } from "../lib/scrape-events";
 import { configDotenv } from "dotenv";
 import {
@@ -66,6 +66,12 @@ export async function runWebScraper({
   is_scrape = false,
   is_crawl = false,
 }: RunWebScraperParams): Promise<ScrapeUrlResponse> {
+  const logger = _logger.child({
+    method: "runWebScraper",
+    module: "runWebscraper",
+    scrapeId: bull_job_id,
+    jobId: bull_job_id,
+  })
   const tries = is_crawl ? 3 : 1;
 
   let response: ScrapeUrlResponse | undefined = undefined;
@@ -75,10 +81,6 @@ export async function runWebScraper({
   for (let i = 0; i < tries; i++) {
     if (i > 0) {
       logger.debug("Retrying scrape...", {
-        scrapeId: bull_job_id,
-        jobId: bull_job_id,
-        method: "runWebScraper",
-        module: "runWebScraper",
         tries,
         i,
         previousStatusCode: (response as any)?.document?.metadata?.statusCode,
@@ -171,9 +173,10 @@ export async function runWebScraper({
         creditsToBeBilled = 5;
       }
 
-      billTeam(team_id, undefined, creditsToBeBilled).catch((error) => {
+      billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {
         logger.error(
-          `Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`,
+          `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
+          { error }
         );
         // Optionally, you could notify an admin or add to a retry queue here
       });
@@ -232,6 +235,11 @@ const saveJob = async (
     }
     ScrapeEvents.logJobEvent(job, "completed");
   } catch (error) {
-    logger.error(`🐂 Failed to update job status: ${error}`);
+    _logger.error(`🐂 Failed to update job status`, {
+      module: "runWebScraper",
+      method: "saveJob",
+      jobId: job.id,
+      scrapeId: job.id,
+    });
   }
 };

apps/api/src/services/billing/credit_billing.ts

@@ -10,6 +10,7 @@ import { issueCredits } from "./issue_credits";
 import { redlock } from "../redlock";
 import { autoCharge } from "./auto_charge";
 import { getValue, setValue } from "../redis";
+import type { Logger } from "winston";
 
 const FREE_CREDITS = 500;
 
@@ -20,22 +21,30 @@ export async function billTeam(
   team_id: string,
   subscription_id: string | null | undefined,
   credits: number,
+  logger?: Logger,
 ) {
   return withAuth(supaBillTeam, { success: true, message: "No DB, bypassed." })(
     team_id,
     subscription_id,
     credits,
+    logger,
   );
 }
 export async function supaBillTeam(
   team_id: string,
   subscription_id: string | null | undefined,
   credits: number,
+  __logger?: Logger,
 ) {
+  const _logger = (__logger ?? logger).child({
+    module: "credit_billing",
+    method: "supaBillTeam",
+  });
+
   if (team_id === "preview") {
     return { success: true, message: "Preview team, no credits used" };
   }
-  logger.info(`Billing team ${team_id} for ${credits} credits`);
+  _logger.info(`Billing team ${team_id} for ${credits} credits`, { team_id, credits });
 
   const { data, error } = await supabase_service.rpc("bill_team", {
     _team_id: team_id,
@@ -46,7 +55,7 @@ export async function supaBillTeam(
 
   if (error) {
     Sentry.captureException(error);
-    logger.error("Failed to bill team: " + JSON.stringify(error));
+    _logger.error("Failed to bill team.", { error });
     return;
   }