AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-17 04:35:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-07-12 23:20:26 -04:00
parent d0c8d3ecde
commit 949791049f
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -47,13 +47,18 @@ export async function scrapWithFireEngine({
const reqParams = await generateRequestParams(url); const reqParams = await generateRequestParams(url);
const waitParam = reqParams["params"]?.wait ?? waitFor; const waitParam = reqParams["params"]?.wait ?? waitFor;
const screenshotParam = reqParams["params"]?.screenshot ?? screenshot; const screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
const fireEngineOptionsParam = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions; const fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions;
let endpoint = fireEngineOptionsParam.method === "get" ? "/request" : "/scrape";
console.log( console.log(
`[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam}` `[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam} and method: ${fireEngineOptionsParam?.method ?? "null"}`
); );
console.log(fireEngineOptionsParam)
const response = await axios.post( const response = await axios.post(
process.env.FIRE_ENGINE_BETA_URL + "/scrape", process.env.FIRE_ENGINE_BETA_URL + endpoint,
{ {
url: url, url: url,
wait: waitParam, wait: waitParam,
@ -74,6 +79,7 @@ export async function scrapWithFireEngine({
console.error( console.error(
`[Fire-Engine] Error fetching url: ${url} with status: ${response.status}` `[Fire-Engine] Error fetching url: ${url} with status: ${response.status}`
); );
logParams.error_message = response.data?.pageError; logParams.error_message = response.data?.pageError;
logParams.response_code = response.data?.pageStatusCode; logParams.response_code = response.data?.pageStatusCode;

10
apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
View File

@ -192,4 +192,14 @@ export const urlSpecificParams = {
"ir.veeva.com":{ "ir.veeva.com":{
defaultScraper: "fire-engine", defaultScraper: "fire-engine",
}, },
"eonhealth.com":{
defaultScraper: "fire-engine",
params:{
fireEngineOptions:{
mobileProxy: true,
method: "get",
engine: "request",
},
},
},
}; };