diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts index 182568fe..a6eca435 100644 --- a/apps/api/src/controllers/v0/scrape.ts +++ b/apps/api/src/controllers/v0/scrape.ts @@ -61,7 +61,7 @@ export async function scrapeHelper( error: "Request timed out", returnCode: 408, } - } else if (typeof e === "string" && (e.includes("Error generating completions: ") || e.includes("Invalid schema for function"))) { + } else if (typeof e === "string" && (e.includes("Error generating completions: ") || e.includes("Invalid schema for function") || e.includes("LLM extraction did not match the extraction schema you provided."))) { return { success: false, error: e, diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts index 42525257..77411b00 100644 --- a/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts +++ b/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts @@ -8,7 +8,6 @@ describe('Blocklist Functionality', () => { 'https://twitter.com/home', 'https://instagram.com/explore', 'https://linkedin.com/in/johndoe', - 'https://pinterest.com/pin/create', 'https://snapchat.com/add/johndoe', 'https://tiktok.com/@johndoe', 'https://reddit.com/r/funny', diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts index c09cc5b3..3d98fedf 100644 --- a/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts +++ b/apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts @@ -8,7 +8,6 @@ describe('isUrlBlocked', () => { 'https://twitter.com/someuser', 'https://instagram.com/someuser', 'https://www.linkedin.com/in/someuser', - 'https://pinterest.com/someuser', 'https://snapchat.com/someuser', 'https://tiktok.com/@someuser', 'https://reddit.com/r/somesubreddit', diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts index 7f1602e1..7b1ee19c 100644 --- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts +++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts @@ -6,7 +6,6 @@ const socialMediaBlocklist = [ 'twitter.com', 'instagram.com', 'linkedin.com', - 'pinterest.com', 'snapchat.com', 'tiktok.com', 'reddit.com',