diff --git a/apps/api/src/__tests__/snips/billing.test.ts b/apps/api/src/__tests__/snips/billing.test.ts index 3a4898ef..8afa5a44 100644 --- a/apps/api/src/__tests__/snips/billing.test.ts +++ b/apps/api/src/__tests__/snips/billing.test.ts @@ -97,7 +97,7 @@ describe("Billing tests", () => { const rc2 = (await creditUsage()).remaining_credits; expect(rc1 - rc2).toBe(12); - }, 300000); + }, 600000); it("bills crawl correctly", async () => { const rc1 = (await creditUsage()).remaining_credits; diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts index bbb17d6b..1ce2211c 100644 --- a/apps/api/src/services/queue-jobs.ts +++ b/apps/api/src/services/queue-jobs.ts @@ -84,6 +84,8 @@ async function addScrapeJobRaw( // logger.error("Error sending notification (concurrency limit reached): ", error); // }); } + + webScraperOptions.concurrencyLimited = true; await _addScrapeJobToConcurrencyQueue( webScraperOptions, diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 96de8d72..52f1f55a 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -923,6 +923,10 @@ async function processJob(job: Job & { id: string }, token: string) { delete doc.rawHtml; } + if (job.data.concurrencyLimited) { + doc.warning = "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." + (doc.warning ? " " + doc.warning : ""); + } + const data = { success: true, result: {