AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-03 04:20:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extractSmartScrape): mog it to 100 pages max

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-16 01:13:49 -07:00
parent 129b10e478
commit dcef6fbc13
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
View File

@ -279,10 +279,18 @@ export async function extractData({
smartScrapeCost += smartscrapeResults[0].tokenUsage; smartScrapeCost += smartscrapeResults[0].tokenUsage;
smartScrapeCallCount++; smartScrapeCallCount++;
} else { } else {
const pages = extract?.smartscrapePages; const pages = extract?.smartscrapePages ?? [];
//do it async promiseall instead //do it async promiseall instead
if (pages.length > 100) {
logger.warn("Smart scrape pages limit exceeded, only first 100 pages will be scraped", {
pagesLength: pages.length,
extractId,
scrapeId,
});
}
smartscrapeResults = await Promise.all( smartscrapeResults = await Promise.all(
pages.map(async (page) => { pages.slice(0, 100).map(async (page) => {
return await smartScrape( return await smartScrape(
urls[page.page_index], urls[page.page_index],
page.smartscrape_prompt, page.smartscrape_prompt,