AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-04 11:24:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(search): ignoreBlockedURLs (FIR-1954) (#1580)

Browse Source 
* feat(search): ignoreBlockedURLs

* change name
This commit is contained in:
Gergő Móricz 2025-05-21 18:02:58 +02:00 committed by GitHub
parent 3d4692f47c
commit 9bb97388e2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/controllers/v1/search.ts
View File

@ -196,6 +196,10 @@ export async function searchController(
location: req.body.location,
});
if (req.body.ignoreInvalidURLs) {
searchResults = searchResults.filter((result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null));
}
logger.info("Searching completed", {
num_results: searchResults.length,
});

1
apps/api/src/controllers/v1/types.ts
View File

@ -1163,6 +1163,7 @@ export const searchRequestSchema = z
location: z.string().optional(),
origin: z.string().optional().default("api"),
timeout: z.number().int().positive().finite().safe().default(60000),
ignoreInvalidURLs: z.boolean().optional().default(false),
scrapeOptions: baseScrapeOptions
.extend({
formats: z