Merge branch 'main' into fix/preview_token

apps/api/src/controllers/auth.ts

@@ -100,7 +100,8 @@ export async function getACUC(
       ? "auth_credit_usage_chunk_extract"
       : "auth_credit_usage_chunk_test_22_credit_pack_n_extract";
     while (retries < maxRetries) {
-      const client = Math.random() > 0.75 ? supabase_rr_service : supabase_service;
+      const client =
+        Math.random() > 0.5 ? supabase_rr_service : supabase_service;
       ({ data, error } = await client.rpc(
         rpcName,
         { input_key: api_key },
@@ -200,6 +201,11 @@ export async function supaAuthenticateUser(
   let priceId: string | null = null;
   let chunk: AuthCreditUsageChunk | null = null;
   let plan: PlanType = "free";
+  if (token == "this_is_just_a_preview_token") {
+    throw new Error(
+      "Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",
+    );
+  }
   if (token == process.env.PREVIEW_TOKEN) {
     if (mode == RateLimiterMode.CrawlStatus) {
       rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);

apps/api/src/controllers/v1/crawl-status.ts

@@ -21,7 +21,7 @@ import {
 import { configDotenv } from "dotenv";
 import type { Job, JobState, Queue } from "bullmq";
 import { logger } from "../../lib/logger";
-import { supabase_service } from "../../services/supabase";
+import { supabase_rr_service, supabase_service } from "../../services/supabase";
 import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
 configDotenv();
 
@@ -246,7 +246,7 @@ export async function crawlStatusController(
   let totalCount = jobIDs.length;
 
   if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {
-    const x = await supabase_service
+    const x = await supabase_rr_service
       .from('firecrawl_jobs')
       .select('*', { count: 'exact', head: true })
       .eq("crawl_id", req.params.jobId)

apps/api/src/lib/extract/team-id-sync.ts

@@ -1,9 +1,9 @@
-import { supabase_service } from "../../services/supabase";
+import { supabase_rr_service, supabase_service } from "../../services/supabase";
 import { logger } from "../logger";
 
 export async function getTeamIdSyncB(teamId: string) {
   try {
-    const { data, error } = await supabase_service
+    const { data, error } = await supabase_rr_service
       .from("eb-sync")
       .select("team_id")
       .eq("team_id", teamId)

apps/api/src/lib/supabase-jobs.ts

@@ -1,4 +1,4 @@
-import { supabase_service } from "../services/supabase";
+import { supabase_rr_service, supabase_service } from "../services/supabase";
 import { logger } from "./logger";
 import * as Sentry from "@sentry/node";
 
@@ -8,7 +8,7 @@ import * as Sentry from "@sentry/node";
  * @returns {any | null} Job
  */
 export const supabaseGetJobById = async (jobId: string) => {
-  const { data, error } = await supabase_service
+  const { data, error } = await supabase_rr_service
     .from("firecrawl_jobs")
     .select("*")
     .eq("job_id", jobId)
@@ -31,7 +31,7 @@ export const supabaseGetJobById = async (jobId: string) => {
  * @returns {any[]} Jobs
  */
 export const supabaseGetJobsById = async (jobIds: string[]) => {
-  const { data, error } = await supabase_service
+  const { data, error } = await supabase_rr_service
     .from("firecrawl_jobs")
     .select()
     .in("job_id", jobIds);
@@ -55,7 +55,7 @@ export const supabaseGetJobsById = async (jobIds: string[]) => {
  * @returns {any[]} Jobs
  */
 export const supabaseGetJobsByCrawlId = async (crawlId: string) => {
-  const { data, error } = await supabase_service
+  const { data, error } = await supabase_rr_service
     .from("firecrawl_jobs")
     .select()
     .eq("crawl_id", crawlId);
@@ -74,7 +74,7 @@ export const supabaseGetJobsByCrawlId = async (crawlId: string) => {
 };
 
 export const supabaseGetJobByIdOnlyData = async (jobId: string) => {
-  const { data, error } = await supabase_service
+  const { data, error } = await supabase_rr_service
     .from("firecrawl_jobs")
     .select("docs, team_id")
     .eq("job_id", jobId)

apps/api/src/scraper/WebScraper/utils/blocklist.ts

@@ -67,6 +67,7 @@ const urlBlocklist = [
   "vMdzZ33BXoyWVZnAPOBcrg==",
   "l8GDVI8w/ueHnNzdN1ODuQ==",
   "+yz9bnYYMnC0trJZGJwf6Q==",
+  "oTdhIjEjqdT2pEvyxD1Ssg==",
 ]
 
 const allowedKeywords = [

apps/api/src/services/billing/auto_charge.ts

@@ -2,7 +2,7 @@
 import { AuthCreditUsageChunk } from "../../controllers/v1/types";
 import { getACUC } from "../../controllers/auth";
 import { redlock } from "../redlock";
-import { supabase_service } from "../supabase";
+import { supabase_rr_service, supabase_service } from "../supabase";
 import { createPaymentIntent } from "./stripe";
 import { issueCredits } from "./issue_credits";
 import { sendNotification, sendNotificationWithCustomDays } from "../notification/email_notification";
@@ -124,7 +124,7 @@ export async function autoCharge(
           if (chunk.sub_user_id) {
             // Fetch the customer's Stripe information
             const { data: customer, error: customersError } =
-              await supabase_service
+              await supabase_rr_service
                 .from("customers")
                 .select("id, stripe_customer_id")
                 .eq("id", chunk.sub_user_id)

apps/api/src/services/billing/batch_billing.ts

@@ -10,8 +10,8 @@ import { getACUC, setCachedACUC } from "../../controllers/auth";
 const BATCH_KEY = "billing_batch";
 const BATCH_LOCK_KEY = "billing_batch_lock";
 const BATCH_SIZE = 50; // Batch size for processing
-const BATCH_TIMEOUT = 15000; // 15 seconds processing interval
+const BATCH_TIMEOUT = 30000; // 15 seconds processing interval
-const LOCK_TIMEOUT = 30000; // 30 seconds lock timeout
+const LOCK_TIMEOUT = 60000; // 30 seconds lock timeout
 
 // Define interfaces for billing operations
 interface BillingOperation {

apps/api/src/services/billing/credit_billing.ts

@@ -1,13 +1,10 @@
 import { NotificationType } from "../../types";
 import { withAuth } from "../../lib/withAuth";
 import { sendNotification } from "../notification/email_notification";
-import { supabase_service } from "../supabase";
+import { supabase_rr_service, supabase_service } from "../supabase";
 import { logger } from "../../lib/logger";
 import * as Sentry from "@sentry/node";
 import { AuthCreditUsageChunk } from "../../controllers/v1/types";
-import { getACUC, setCachedACUC } from "../../controllers/auth";
-import { issueCredits } from "./issue_credits";
-import { redlock } from "../redlock";
 import { autoCharge } from "./auto_charge";
 import { getValue, setValue } from "../redis";
 import { queueBillingOperation } from "./batch_billing";
@@ -117,7 +114,7 @@ export async function supaCheckTeamCredits(
     isAutoRechargeEnabled = parsedData.auto_recharge;
     autoRechargeThreshold = parsedData.auto_recharge_threshold;
   } else {
-    const { data, error } = await supabase_service
+    const { data, error } = await supabase_rr_service
       .from("teams")
       .select("auto_recharge, auto_recharge_threshold")
       .eq("id", team_id)

apps/api/src/services/idempotency/validate.ts

@@ -1,5 +1,5 @@
 import { Request } from "express";
-import { supabase_service } from "../supabase";
+import { supabase_rr_service, supabase_service } from "../supabase";
 import { validate as isUuid } from "uuid";
 import { logger } from "../../../src/lib/logger";
 
@@ -18,7 +18,7 @@ export async function validateIdempotencyKey(req: Request): Promise<boolean> {
     return false;
   }
 
-  const { data, error } = await supabase_service
+  const { data, error } = await supabase_rr_service
     .from("idempotency_keys")
     .select("key")
     .eq("key", idempotencyKey);

apps/api/src/services/rate-limiter.ts

@@ -137,8 +137,8 @@ const RATE_LIMITS = {
     extract_pro: 1000,
   },
   preview: {
-    free: 5,
+    free: 0,
-    default: 5,
+    default: 0,
   },
   account: {
     free: 100,

apps/api/src/services/webhook.ts

@@ -1,6 +1,6 @@
 import axios from "axios";
 import { logger } from "../lib/logger";
-import { supabase_service } from "./supabase";
+import { supabase_rr_service, supabase_service } from "./supabase";
 import { WebhookEventType } from "../types";
 import { configDotenv } from "dotenv";
 import { z } from "zod";
@@ -36,7 +36,7 @@ export const callWebhook = async (
     // Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set
     // and the USE_DB_AUTHENTICATION environment variable is set to true
     if (!webhookUrl && useDbAuthentication) {
-      const { data: webhooksData, error } = await supabase_service
+      const { data: webhooksData, error } = await supabase_rr_service
         .from("webhooks")
         .select("url")
         .eq("team_id", teamId)