AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-04-19 12:39:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(queue-jobs): add function to determine job type and update notification logic for concurrency limits

Browse Source
This commit is contained in:
Ademílson F. Tonato 2025-04-03 17:02:51 +01:00
parent 8c1579df51
commit 426151c9c9
No known key found for this signature in database
GPG Key ID: 169C7BE271C9FA3A
1 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
apps/api/src/services/queue-jobs.ts
View File

@ -14,6 +14,17 @@ import { getConcurrencyLimitMax } from "./rate-limiter";
import { sendNotificationWithCustomDays } from './notification/email_notification'; import { sendNotificationWithCustomDays } from './notification/email_notification';
import { shouldSendConcurrencyLimitNotification } from './notification/notification-check'; import { shouldSendConcurrencyLimitNotification } from './notification/notification-check';
/**
* Checks if a job is a crawl or batch scrape based on its options
* @param options The job options containing crawlerOptions and crawl_id
* @returns true if the job is either a crawl or batch scrape
*/
function isCrawlOrBatchScrape(options: { crawlerOptions?: any; crawl_id?: string }): boolean {
// If crawlerOptions exists, it's a crawl
// If crawl_id exists but no crawlerOptions, it's a batch scrape
return !!options.crawlerOptions || !!options.crawl_id;
}
async function _addScrapeJobToConcurrencyQueue( async function _addScrapeJobToConcurrencyQueue(
webScraperOptions: any, webScraperOptions: any,
options: any, options: any,
@ -83,11 +94,14 @@ async function addScrapeJobRaw(
if(concurrencyQueueJobs > maxConcurrency) { if(concurrencyQueueJobs > maxConcurrency) {
logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id); logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);
const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id); // Only send notification if it's not a crawl or batch scrape
if (shouldSendNotification) { if (!isCrawlOrBatchScrape(webScraperOptions)) {
sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => { const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id);
logger.error("Error sending notification (concurrency limit reached): ", error); if (shouldSendNotification) {
}); sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {
logger.error("Error sending notification (concurrency limit reached): ", error);
});
}
} }
} }
@ -178,11 +192,14 @@ export async function addScrapeJobs(
if(addToCQ.length > maxConcurrency) { if(addToCQ.length > maxConcurrency) {
logger.info("Concurrency limited 2x (multiple) - ", "Concurrency queue jobs: ", addToCQ.length, "Max concurrency: ", maxConcurrency, "Team ID: ", jobs[0].data.team_id); logger.info("Concurrency limited 2x (multiple) - ", "Concurrency queue jobs: ", addToCQ.length, "Max concurrency: ", maxConcurrency, "Team ID: ", jobs[0].data.team_id);
const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id); // Only send notification if it's not a crawl or batch scrape
if (shouldSendNotification) { if (!isCrawlOrBatchScrape(jobs[0].data)) {
sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => { const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);
logger.error("Error sending notification (concurrency limit reached): ", error); if (shouldSendNotification) {
}); sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {
logger.error("Error sending notification (concurrency limit reached): ", error);
});
}
} }
} }