AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-04 20:00:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #649 from mendableai/nsc/chrome-cdp-for-params

Browse Source 
Better engine for wait + other params
This commit is contained in:
Nicolas 2024-09-09 21:12:46 -03:00 committed by GitHub
commit 2cbc4c59ce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -55,7 +55,7 @@ export async function scrapWithFireEngine({
try { try {
const reqParams = await generateRequestParams(url); const reqParams = await generateRequestParams(url);
let waitParam = reqParams["params"]?.wait ?? waitFor; let waitParam = reqParams["params"]?.wait ?? waitFor;
let engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "playwright"; let engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "chrome-cdp";
let screenshotParam = reqParams["params"]?.screenshot ?? screenshot; let screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
let fullPageScreenshotParam = reqParams["params"]?.fullPageScreenshot ?? fullPageScreenshot; let fullPageScreenshotParam = reqParams["params"]?.fullPageScreenshot ?? fullPageScreenshot;
let fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions; let fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions;

18
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -96,15 +96,15 @@ function getScrapingFallbackOrder(
"fetch", "fetch",
].filter(Boolean); ].filter(Boolean);
if (isWaitPresent || isScreenshotPresent || isHeadersPresent) { // if (isWaitPresent || isScreenshotPresent || isHeadersPresent) {
defaultOrder = [ // defaultOrder = [
"fire-engine", // "fire-engine",
useFireEngine ? undefined : "playwright", // useFireEngine ? undefined : "playwright",
...defaultOrder.filter( // ...defaultOrder.filter(
(scraper) => scraper !== "fire-engine" && scraper !== "playwright" // (scraper) => scraper !== "fire-engine" && scraper !== "playwright"
), // ),
].filter(Boolean); // ].filter(Boolean);
} // }
const filteredDefaultOrder = defaultOrder.filter( const filteredDefaultOrder = defaultOrder.filter(
(scraper: (typeof baseScrapers)[number]) => (scraper: (typeof baseScrapers)[number]) =>