diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 4da56619..322053d7 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -22,7 +22,7 @@ export const urlSpecificParams = { }, }, "support.greenpay.me":{ - defaultScraper: "playwright", + defaultScraper: "fire-engine", params: { wait_browser: "networkidle2", block_resources: false, @@ -43,7 +43,7 @@ export const urlSpecificParams = { }, }, "docs.pdw.co":{ - defaultScraper: "playwright", + defaultScraper: "fire-engine", params: { wait_browser: "networkidle2", block_resources: false, @@ -83,7 +83,7 @@ export const urlSpecificParams = { }, }, "developers.notion.com":{ - defaultScraper: "playwright", + defaultScraper: "fire-engine", params: { wait_browser: "networkidle2", block_resources: false, @@ -103,7 +103,7 @@ export const urlSpecificParams = { }, }, "docs2.hubitat.com":{ - defaultScraper: "playwright", + defaultScraper: "fire-engine", params: { wait_browser: "networkidle2", block_resources: false, @@ -153,7 +153,7 @@ export const urlSpecificParams = { }, }, "help.salesforce.com":{ - defaultScraper: "playwright", + defaultScraper: "fire-engine", params: { wait_browser: "networkidle2", block_resources: false, @@ -203,4 +203,26 @@ export const urlSpecificParams = { }, }, }, + "notion.com":{ + defaultScraper: "fire-engine", + params: { + wait_browser: "networkidle2", + block_resources: false, + wait: 2000, + engine: "playwright", + }, + headers: { + "User-Agent": + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "sec-fetch-site": "same-origin", + "sec-fetch-mode": "cors", + "sec-fetch-dest": "empty", + referer: "https://www.google.com/", + "accept-language": "en-US,en;q=0.9", + "accept-encoding": "gzip, deflate, br", + accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + }, + }, + };