AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 04:39:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into v1-webscraper

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-23 19:38:57 +02:00
commit 05c250d3b8
4 changed files with 1 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v0/scrape.ts
View File

@ -61,7 +61,7 @@ export async function scrapeHelper(
error: "Request timed out",
returnCode: 408,
}
} else if (typeof e === "string" && (e.includes("Error generating completions: ") || e.includes("Invalid schema for function"))) {
} else if (typeof e === "string" && (e.includes("Error generating completions: ") || e.includes("Invalid schema for function") || e.includes("LLM extraction did not match the extraction schema you provided."))) {
return {
success: false,
error: e,

1
apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts
View File

@ -8,7 +8,6 @@ describe('Blocklist Functionality', () => {
'https://twitter.com/home',
'https://instagram.com/explore',
'https://linkedin.com/in/johndoe',
'https://pinterest.com/pin/create',
'https://snapchat.com/add/johndoe',
'https://tiktok.com/@johndoe',
'https://reddit.com/r/funny',

1
apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts
View File

@ -8,7 +8,6 @@ describe('isUrlBlocked', () => {
'https://twitter.com/someuser',
'https://instagram.com/someuser',
'https://www.linkedin.com/in/someuser',
'https://pinterest.com/someuser',
'https://snapchat.com/someuser',
'https://tiktok.com/@someuser',
'https://reddit.com/r/somesubreddit',

1
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -6,7 +6,6 @@ const socialMediaBlocklist = [
'twitter.com',
'instagram.com',
'linkedin.com',
'pinterest.com',
'snapchat.com',
'tiktok.com',
'reddit.com',