From e1e39f8836a48d6d1d1a921028850f452b1d58ca Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 4 Apr 2025 14:34:48 -0400 Subject: [PATCH] Nick: send notifications for crawl+batch scrape --- apps/api/src/services/notification/email_notification.ts | 2 +- apps/api/src/services/queue-jobs.ts | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts index 58a38cc0..d17896e8 100644 --- a/apps/api/src/services/notification/email_notification.ts +++ b/apps/api/src/services/notification/email_notification.ts @@ -44,7 +44,7 @@ const emailTemplates: Record<

We've improved our system by transitioning to concurrency limits, allowing faster scraping by default and eliminating* the often rate limit errors.

You're hitting the concurrency limit for your plan quite often, which means Firecrawl can't scrape as fast as it could. But don't worry, it is not failing your requests and you are still getting your results.

-

This is just to let you know that you could be scraping more pages faster. Consider upgrading your plan at firecrawl.dev/pricing.


Thanks,
Firecrawl Team
`, +

This is just to let you know that you could be scraping faster. Consider upgrading your plan at firecrawl.dev/pricing.


Thanks,
Firecrawl Team
`, }, }; diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts index 1b4530b9..0a1a49fb 100644 --- a/apps/api/src/services/queue-jobs.ts +++ b/apps/api/src/services/queue-jobs.ts @@ -95,14 +95,12 @@ async function addScrapeJobRaw( logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id); // Only send notification if it's not a crawl or batch scrape - if (!isCrawlOrBatchScrape(webScraperOptions)) { const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id); if (shouldSendNotification) { sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => { logger.error("Error sending notification (concurrency limit reached): ", error); }); } - } } webScraperOptions.concurrencyLimited = true; @@ -193,14 +191,12 @@ export async function addScrapeJobs( logger.info("Concurrency limited 2x (multiple) - ", "Concurrency queue jobs: ", addToCQ.length, "Max concurrency: ", maxConcurrency, "Team ID: ", jobs[0].data.team_id); // Only send notification if it's not a crawl or batch scrape - if (!isCrawlOrBatchScrape(jobs[0].data)) { const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id); if (shouldSendNotification) { sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => { logger.error("Error sending notification (concurrency limit reached): ", error); }); } - } } await Promise.all(