AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 09:28:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrape): add warning to document if it was concurrency limited (#1348)

Browse Source 
* feat(scrape): add warning to document if it was concurrency limited

* phrasing + test fix
This commit is contained in:
Gergő Móricz 2025-03-16 19:57:27 +01:00 committed by GitHub
parent 670ca84ae9
commit 200de9e7e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/__tests__/snips/billing.test.ts
View File

@ -97,7 +97,7 @@ describe("Billing tests", () => {
const rc2 = (await creditUsage()).remaining_credits; const rc2 = (await creditUsage()).remaining_credits;
expect(rc1 - rc2).toBe(12); expect(rc1 - rc2).toBe(12);
}, 300000); }, 600000);
it("bills crawl correctly", async () => { it("bills crawl correctly", async () => {
const rc1 = (await creditUsage()).remaining_credits; const rc1 = (await creditUsage()).remaining_credits;

2
apps/api/src/services/queue-jobs.ts
View File

@ -85,6 +85,8 @@ async function addScrapeJobRaw(
// }); // });
} }
webScraperOptions.concurrencyLimited = true;
await _addScrapeJobToConcurrencyQueue( await _addScrapeJobToConcurrencyQueue(
webScraperOptions, webScraperOptions,
options, options,

4
apps/api/src/services/queue-worker.ts
View File

@ -923,6 +923,10 @@ async function processJob(job: Job & { id: string }, token: string) {
delete doc.rawHtml; delete doc.rawHtml;
} }
if (job.data.concurrencyLimited) {
doc.warning = "This scrape job was throttled at your current concurrency limit. If you'd like to scrape faster, you can upgrade your plan." + (doc.warning ? " " + doc.warning : "");
}
const data = { const data = {
success: true, success: true,
result: { result: {