diff --git a/apps/api/src/controllers/crawl.ts b/apps/api/src/controllers/crawl.ts index bf402d38..1dfe758f 100644 --- a/apps/api/src/controllers/crawl.ts +++ b/apps/api/src/controllers/crawl.ts @@ -56,14 +56,17 @@ export async function crawlController(req: Request, res: Response) { }; const pageOptions = { ...defaultCrawlPageOptions, ...req.body.pageOptions }; - const limitCheck = crawlerOptions?.limit ?? 1; - const { success: creditsCheckSuccess, message: creditsCheckMessage } = + const limitCheck = req.body?.crawlerOptions?.limit ?? 1; + const { success: creditsCheckSuccess, message: creditsCheckMessage, remainingCredits } = await checkTeamCredits(team_id, limitCheck); if (!creditsCheckSuccess) { - return res.status(402).json({ error: "Insufficient credits" }); + return res.status(402).json({ error: "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at hello@firecrawl.com" }); } + // TODO: need to do this to v1 + crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit); + let url = req.body.url; if (!url) { return res.status(400).json({ error: "Url is required" }); diff --git a/apps/api/src/lib/default-values.ts b/apps/api/src/lib/default-values.ts index 152f47d7..0b469ee2 100644 --- a/apps/api/src/lib/default-values.ts +++ b/apps/api/src/lib/default-values.ts @@ -12,7 +12,8 @@ export const defaultPageOptions = { }; export const defaultCrawlerOptions = { - allowBackwardCrawling: false + allowBackwardCrawling: false, + limit: 10000 } export const defaultCrawlPageOptions = { diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index d25289b2..2ad07318 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -168,10 +168,11 @@ export async function supaBillTeam(team_id: string, credits: number) { export async function checkTeamCredits(team_id: string, credits: number) { return withAuth(supaCheckTeamCredits)(team_id, credits); } + // if team has enough credits for the operation, return true, else return false export async function supaCheckTeamCredits(team_id: string, credits: number) { if (team_id === "preview") { - return { success: true, message: "Preview team, no credits used" }; + return { success: true, message: "Preview team, no credits used", remainingCredits: Infinity }; } // Retrieve the team's active subscription and check for available coupons concurrently @@ -202,7 +203,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { if (subscriptionError || !subscription) { // If there is no active subscription but there are available coupons if (couponCredits >= credits) { - return { success: true, message: "Sufficient credits available" }; + return { success: true, message: "Sufficient credits available", remainingCredits: couponCredits }; } const { data: creditUsages, error: creditUsageError } = @@ -252,9 +253,10 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { return { success: false, message: "Insufficient credits, please upgrade!", + remainingCredits: FREE_CREDITS - totalCreditsUsed }; } - return { success: true, message: "Sufficient credits available" }; + return { success: true, message: "Sufficient credits available", remainingCredits: FREE_CREDITS - totalCreditsUsed }; } let totalCreditsUsed = 0; @@ -321,7 +323,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { subscription.current_period_start, subscription.current_period_end ); - return { success: false, message: "Insufficient credits, please upgrade!" }; + return { success: false, message: "Insufficient credits, please upgrade!", remainingCredits: creditLimit - adjustedCreditsUsed }; } else if (creditUsagePercentage >= 0.8) { // Send email notification for approaching credit limit await sendNotification( @@ -332,7 +334,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { ); } - return { success: true, message: "Sufficient credits available" }; + return { success: true, message: "Sufficient credits available", remainingCredits: creditLimit - adjustedCreditsUsed }; } // Count the total credits used by a team within the current billing period and return the remaining credits. diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 0d24387a..6e848ce6 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -117,13 +117,13 @@ async function processJob(job: Job, token: string) { // Check if the job URL is researchhub and block it immediately // TODO: remove this once solve the root issue - if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com") || job.data.url.includes("youtube.com"))) { + if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com") || job.data.url.includes("youtube.com") || job.data.url.includes("microsoft.com"))) { Logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`); const data = { success: false, docs: [], project_id: job.data.project_id, - error: "URL is blocked. Please contact hello@firecrawl.com if you believe this is an error.", + error: "URL is blocked. Suspecious activity detected. Please contact hello@firecrawl.com if you believe this is an error.", }; await job.moveToCompleted(data.docs, token, false); return data;