AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-15 00:45:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v1-webscraper' of https://github.com/mendableai/firecrawl into v1-webscraper

Browse Source
This commit is contained in:
Nicolas 2024-08-26 18:17:17 -03:00
commit 8c37ea6d96
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/controllers/v1/types.ts
View File

@ -27,7 +27,10 @@ const url = z.preprocess(
.string() .string()
.url() .url()
.regex(/^https?:\/\//, "URL uses unsupported protocol") .regex(/^https?:\/\//, "URL uses unsupported protocol")
.regex(/\.[a-z]{2,}$/i, "URL must have a valid top-level domain") .refine(
(x) => /\.[a-z]{2,}(\/|$)/i.test(x),
"URL must have a valid top-level domain or be a valid path"
)
.refine( .refine(
(x) => !isUrlBlocked(x), (x) => !isUrlBlocked(x),
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it."