From 9edbdc987fe910f7903fb5bceb07cb4692368d7e Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 12 Mar 2025 14:17:22 -0400 Subject: [PATCH 1/3] Update rate-limiter.ts --- apps/api/src/services/rate-limiter.ts | 190 +++++++++++++------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 6aecffee..719cbf85 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -28,133 +28,133 @@ export const CONCURRENCY_LIMIT: Omit, ""> = { const RATE_LIMITS = { crawl: { - default: 3, + default: 15, free: 2, - starter: 10, - standard: 5, - standardOld: 40, - scale: 50, - hobby: 3, - standardNew: 10, - standardnew: 10, - growth: 50, - growthdouble: 50, - etier2c: 300, - etier1a: 1000, - etier2a: 300, - etierscale1: 150, - etierscale2: 300, + starter: 50, + standard: 25, + standardOld: 200, + scale: 250, + hobby: 15, + standardNew: 50, + standardnew: 50, + growth: 250, + growthdouble: 250, + etier2c: 1500, + etier1a: 5000, + etier2a: 1500, + etierscale1: 750, + etierscale2: 1500, // extract ops - extract_starter: 20, - extract_explorer: 100, - extract_pro: 1000, + extract_starter: 100, + extract_explorer: 500, + extract_pro: 5000, }, scrape: { - default: 20, + default: 100, free: 10, - starter: 100, - standard: 100, - standardOld: 100, - scale: 500, - hobby: 20, - standardNew: 100, - standardnew: 100, - growth: 1000, - growthdouble: 1000, - etier2c: 2500, - etier1a: 1000, - etier2a: 2500, - etierscale1: 1500, - etierscale2: 2500, + starter: 500, + standard: 500, + standardOld: 500, + scale: 2500, + hobby: 100, + standardNew: 500, + standardnew: 500, + growth: 5000, + growthdouble: 5000, + etier2c: 12500, + etier1a: 5000, + etier2a: 12500, + etierscale1: 7500, + etierscale2: 12500, // extract ops - extract_starter: 20, - extract_explorer: 100, - extract_pro: 1000, + extract_starter: 100, + extract_explorer: 500, + extract_pro: 5000, }, search: { - default: 20, + default: 100, free: 5, - starter: 50, - standard: 50, - standardOld: 40, - scale: 500, - hobby: 10, - standardNew: 50, - standardnew: 50, - growth: 500, - growthdouble: 500, - etier2c: 2500, - etier1a: 1000, - etier2a: 2500, - etierscale1: 1500, - etierscale2: 2500, + starter: 250, + standard: 250, + standardOld: 200, + scale: 2500, + hobby: 50, + standardNew: 250, + standardnew: 250, + growth: 2500, + growthdouble: 2500, + etier2c: 12500, + etier1a: 5000, + etier2a: 12500, + etierscale1: 7500, + etierscale2: 12500, // extract ops - extract_starter: 20, - extract_explorer: 100, - extract_pro: 1000, + extract_starter: 100, + extract_explorer: 500, + extract_pro: 5000, }, map: { - default: 20, + default: 100, free: 5, - starter: 50, - standard: 50, - standardOld: 50, - scale: 500, - hobby: 10, - standardNew: 50, - standardnew: 50, - growth: 1000, - growthdouble: 1000, - etier2c: 2500, - etier1a: 1000, - etier2a: 2500, - etierscale1: 1500, - etierscale2: 2500, + starter: 250, + standard: 250, + standardOld: 250, + scale: 2500, + hobby: 50, + standardNew: 250, + standardnew: 250, + growth: 5000, + growthdouble: 5000, + etier2c: 12500, + etier1a: 5000, + etier2a: 12500, + etierscale1: 7500, + etierscale2: 12500, // extract ops - extract_starter: 20, - extract_explorer: 100, - extract_pro: 1000, + extract_starter: 100, + extract_explorer: 500, + extract_pro: 5000, }, extract: { - default: 20, + default: 100, free: 10, - starter: 100, - standard: 100, - standardOld: 100, - scale: 300, - hobby: 20, - standardNew: 100, - standardnew: 100, - growth: 300, - growthdouble: 300, - etier2c: 1000, - etier1a: 1000, - etier2a: 1000, - etierscale1: 1000, - etierscale2: 1000, - extract_starter: 20, - extract_explorer: 100, - extract_pro: 1000, + starter: 500, + standard: 500, + standardOld: 500, + scale: 1500, + hobby: 100, + standardNew: 500, + standardnew: 500, + growth: 1500, + growthdouble: 1500, + etier2c: 5000, + etier1a: 5000, + etier2a: 5000, + etierscale1: 5000, + etierscale2: 5000, + extract_starter: 100, + extract_explorer: 500, + extract_pro: 5000, }, preview: { free: 5, - default: 5, + default: 25, }, account: { free: 100, - default: 100, + default: 500, }, crawlStatus: { free: 500, - default: 5000, + default: 25000, }, extractStatus: { free: 500, - default: 5000, + default: 25000, }, testSuite: { free: 10000, - default: 10000, + default: 50000, }, }; From d438b235f6c70ab6050e8a8203ea953bf040598f Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 12 Mar 2025 15:22:28 -0400 Subject: [PATCH 2/3] Update rate-limiter.ts --- apps/api/src/services/rate-limiter.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 719cbf85..6d159fbe 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -47,7 +47,7 @@ const RATE_LIMITS = { // extract ops extract_starter: 100, extract_explorer: 500, - extract_pro: 5000, + extract_pro: 1000, }, scrape: { default: 100, @@ -69,7 +69,7 @@ const RATE_LIMITS = { // extract ops extract_starter: 100, extract_explorer: 500, - extract_pro: 5000, + extract_pro: 1000, }, search: { default: 100, @@ -91,7 +91,7 @@ const RATE_LIMITS = { // extract ops extract_starter: 100, extract_explorer: 500, - extract_pro: 5000, + extract_pro: 1000, }, map: { default: 100, @@ -113,7 +113,7 @@ const RATE_LIMITS = { // extract ops extract_starter: 100, extract_explorer: 500, - extract_pro: 5000, + extract_pro: 1000, }, extract: { default: 100, @@ -121,20 +121,20 @@ const RATE_LIMITS = { starter: 500, standard: 500, standardOld: 500, - scale: 1500, + scale: 1000, hobby: 100, standardNew: 500, standardnew: 500, - growth: 1500, - growthdouble: 1500, - etier2c: 5000, - etier1a: 5000, - etier2a: 5000, - etierscale1: 5000, - etierscale2: 5000, + growth: 1000, + growthdouble: 1000, + etier2c: 1000, + etier1a: 1000, + etier2a: 1000, + etierscale1: 1000, + etierscale2: 1000, extract_starter: 100, extract_explorer: 500, - extract_pro: 5000, + extract_pro: 1000, }, preview: { free: 5, From e8861893948620fdc37457dfc919d031259a38e3 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 12 Mar 2025 20:21:19 -0400 Subject: [PATCH 3/3] Update search.ts --- apps/api/src/controllers/v1/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts index 7975e75a..18ff9579 100644 --- a/apps/api/src/controllers/v1/search.ts +++ b/apps/api/src/controllers/v1/search.ts @@ -139,7 +139,7 @@ export async function searchController( let limit = req.body.limit; // Buffer results by 50% to account for filtered URLs - const num_results_buffer = Math.floor(limit * 1.5); + const num_results_buffer = Math.floor(limit * 2); let searchResults = await search({ query: req.body.query,