AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 03:09:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update search.ts

Browse Source
This commit is contained in:
Nicolas 2025-01-02 20:40:38 -03:00
parent b244afbc82
commit 7ce780ac81
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/controllers/v1/search.ts
View File

@ -112,7 +112,6 @@ export async function searchController(
}); });
// Filter blocked URLs early to avoid unnecessary billing // Filter blocked URLs early to avoid unnecessary billing
searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));
if (searchResults.length > limit) { if (searchResults.length > limit) {
searchResults = searchResults.slice(0, limit); searchResults = searchResults.slice(0, limit);
} }
@ -152,7 +151,10 @@ export async function searchController(
}); });
} }
// Scrape each result, handling timeouts individually // Filter out blocked URLs before scraping
searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));
// Scrape each non-blocked result, handling timeouts individually
const scrapePromises = searchResults.map((result) => const scrapePromises = searchResults.map((result) =>
scrapeSearchResult(result, { scrapeSearchResult(result, {
teamId: req.auth.team_id, teamId: req.auth.team_id,