feat(search): ignore concurrency limit for search (FIR-2187) (#1617)

* feat(search): ignore concurrency limit for search (temp)

* feat(search): only for low tier users for good DX

apps/api/src/controllers/v1/search.ts

@@ -76,6 +76,7 @@ async function scrapeSearchResult(
   logger: Logger,
   costTracking: CostTracking,
   flags: TeamFlags,
+  directToBullMQ: boolean = false,
 ): Promise<Document> {
   const jobId = uuidv4();
   const jobPriority = await getJobPriority({
@@ -102,11 +103,11 @@ async function scrapeSearchResult(
         internalOptions: { teamId: options.teamId, useCache: true },
         origin: options.origin,
         is_scrape: true,
-        
       },
       {},
       jobId,
       jobPriority,
+      directToBullMQ,
     );
 
     const doc: Document = await waitForJob(jobId, options.timeout);
@@ -229,7 +230,7 @@ export async function searchController(
           origin: req.body.origin,
           timeout: req.body.timeout,
           scrapeOptions: req.body.scrapeOptions,
-        }, logger, costTracking, req.acuc?.flags ?? null),
+        }, logger, costTracking, req.acuc?.flags ?? null, (req.acuc?.price_credits ?? 0) <= 3000),
       );
 
       const docs = await Promise.all(scrapePromises);

apps/api/src/services/queue-jobs.ts

@@ -97,6 +97,7 @@ async function addScrapeJobRaw(
   options: any,
   jobId: string,
   jobPriority: number,
+  directToBullMQ: boolean = false,
 ) {
   const hasCrawlDelay = webScraperOptions.crawl_id && webScraperOptions.crawlerOptions?.delay;
 
@@ -127,7 +128,7 @@ async function addScrapeJobRaw(
 
   const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(webScraperOptions.team_id);
 
-  if (concurrencyLimited) {
+  if (concurrencyLimited && !directToBullMQ) {
     // Detect if they hit their concurrent limit
     // If above by 2x, send them an email
     // No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x
@@ -161,6 +162,7 @@ export async function addScrapeJob(
   options: any = {},
   jobId: string = uuidv4(),
   jobPriority: number = 10,
+  directToBullMQ: boolean = false,
 ) {
   if (Sentry.isInitialized()) {
     const size = JSON.stringify(webScraperOptions).length;
@@ -187,6 +189,7 @@ export async function addScrapeJob(
           options,
           jobId,
           jobPriority,
+          directToBullMQ,
         );
       },
     );