Nick: bypass billing

apps/api/src/controllers/v1/scrape.ts

@@ -54,6 +54,7 @@ export async function scrapeController(
         saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
         unnormalizedSourceURL: preNormalizedBody.url,
         useCache: req.body.__experimental_cache ? true : false,
+        bypassBilling: isDirectToBullMQ,
       },
       origin: req.body.origin,
       startTime,
@@ -133,6 +134,7 @@ export async function scrapeController(
     }
   }
 
+
   return res.status(200).json({
     success: true,
     data: doc,

apps/api/src/controllers/v1/search.ts

@@ -40,23 +40,23 @@ export async function searchAndScrapeSearchResult(
   try {
     const searchResults = await search({
       query,
-      num_results: 5
+      num_results: 5,
     });
 
     const documents = await Promise.all(
-    searchResults.map(result => 
+      searchResults.map((result) =>
         scrapeSearchResult(
           {
             url: result.url,
             title: result.title,
-          description: result.description
+            description: result.description,
           },
           options,
           logger,
           costTracking,
-        flags
+          flags,
-      )
+        ),
-    )
+      ),
     );
 
     return documents;
@@ -171,6 +171,8 @@ export async function searchController(
   };
   const startTime = new Date().getTime();
   const costTracking = new CostTracking();
+  const isSearchPreview =
+    process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
 
   try {
     req.body = searchRequestSchema.parse(req.body);
@@ -199,7 +201,9 @@ export async function searchController(
     });
 
     if (req.body.ignoreInvalidURLs) {
-      searchResults = searchResults.filter((result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null));
+      searchResults = searchResults.filter(
+        (result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null),
+      );
     }
 
     logger.info("Searching completed", {
@@ -226,12 +230,19 @@ export async function searchController(
     } else {
       logger.info("Scraping search results");
       const scrapePromises = searchResults.map((result) =>
-        scrapeSearchResult(result, {
+        scrapeSearchResult(
+          result,
+          {
             teamId: req.auth.team_id,
             origin: req.body.origin,
             timeout: req.body.timeout,
             scrapeOptions: req.body.scrapeOptions,
-        }, logger, costTracking, req.acuc?.flags ?? null, (req.acuc?.price_credits ?? 0) <= 3000),
+          },
+          logger,
+          costTracking,
+          req.acuc?.flags ?? null,
+          (req.acuc?.price_credits ?? 0) <= 3000,
+        ),
       );
 
       const docs = await Promise.all(scrapePromises);
@@ -257,17 +268,23 @@ export async function searchController(
     }
 
     // Bill team once for all successful results
-    billTeam(req.auth.team_id, req.acuc?.sub_id, responseData.data.reduce((a,x) => {
+    if (!isSearchPreview) {
+      billTeam(
+        req.auth.team_id,
+        req.acuc?.sub_id,
+        responseData.data.reduce((a, x) => {
           if (x.metadata?.numPages !== undefined && x.metadata.numPages > 0) {
             return a + x.metadata.numPages;
           } else {
             return a + 1;
           }
-    }, 0)).catch((error) => {
+        }, 0),
+      ).catch((error) => {
         logger.error(
           `Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,
         );
       });
+    }
 
     const endTime = new Date().getTime();
     const timeTakenInSeconds = (endTime - startTime) / 1000;
@@ -277,7 +294,8 @@ export async function searchController(
       time_taken: timeTakenInSeconds,
     });
 
-    logJob({
+    logJob(
+      {
         job_id: jobId,
         success: true,
         num_docs: responseData.data.length,
@@ -289,10 +307,12 @@ export async function searchController(
         scrapeOptions: req.body.scrapeOptions,
         origin: req.body.origin,
         cost_tracking: costTracking,
-    });
+      },
+      false,
+      isSearchPreview,
+    );
 
     return res.status(200).json(responseData);
-
   } catch (error) {
     if (
       error instanceof Error &&

apps/api/src/controllers/v1/types.ts

@@ -1169,6 +1169,7 @@ export const searchRequestSchema = z
     origin: z.string().optional().default("api"),
     timeout: z.number().int().positive().finite().safe().default(60000),
     ignoreInvalidURLs: z.boolean().optional().default(false),
+    __searchPreviewToken: z.string().optional(),
     scrapeOptions: baseScrapeOptions
       .extend({
         formats: z

apps/api/src/scraper/scrapeURL/index.ts

@@ -189,6 +189,7 @@ export type InternalOptions = {
   unnormalizedSourceURL?: string;
 
   saveScrapeResultToGCS?: boolean; // Passed along to fire-engine
+  bypassBilling?: boolean;
 };
 
 export type EngineResultsTracker = {

apps/api/src/services/logging/log_job.ts

@@ -21,13 +21,14 @@ function cleanOfNull<T>(x: T): T {
   }
 }
 
-export async function logJob(job: FirecrawlJob, force: boolean = false) {
+export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLogging: boolean = false) {
   try {
     const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
     if (!useDbAuthentication) {
       return;
     }
     
+
     // Redact any pages that have an authorization header
     // actually, Don't. we use the db to retrieve results now. this breaks authed crawls - mogery
     // if (
@@ -70,6 +71,10 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {
       await saveJobToGCS(job);
     }
 
+    if (bypassLogging) {
+      return;
+    }
+
     if (force) {
       let i = 0,
         done = false;

apps/api/src/services/queue-worker.ts

@@ -319,7 +319,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
         scrapeOptions: sc.scrapeOptions,
         crawlerOptions: sc.crawlerOptions,
         origin: job.data.origin,
-      });
+      }, false, job.data.internalOptions?.bypassBilling ?? false);
       logger.info("Logged crawl!");
 
       const data = {
@@ -371,8 +371,10 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
           origin: job.data.origin,
         },
         true,
+        job.data.internalOptions?.bypassBilling ?? false,
       );
 
+
       // v1 web hooks, call when done with no data, but with event completed
       if (job.data.v1 && job.data.webhook) {
         callWebhook(
@@ -1048,7 +1050,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {
 async function billScrapeJob(job: Job & { id: string }, document: Document, logger: Logger, costTracking?: CostTracking) {
   let creditsToBeBilled: number | null = null;
 
-  if (job.data.is_scrape !== true) {
+  if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
     creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, document, job.id, costTracking);
 
     if (
@@ -1378,6 +1380,7 @@ async function processJob(job: Job & { id: string }, token: string) {
           credits_billed,
         },
         true,
+        job.data.internalOptions?.bypassBilling ?? false,
       );
 
       if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
@@ -1424,7 +1427,7 @@ async function processJob(job: Job & { id: string }, token: string) {
         cost_tracking: costTracking,
         pdf_num_pages: doc.metadata.numPages,
         credits_billed,
-      });
+      }, false, job.data.internalOptions?.bypassBilling ?? false);
     }
 
     logger.info(`🐂 Job done ${job.id}`);
@@ -1523,6 +1526,7 @@ async function processJob(job: Job & { id: string }, token: string) {
         cost_tracking: costTracking,
       },
       true,
+      job.data.internalOptions?.bypassBilling ?? false,
     );
     return data;
   }