AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 03:49:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-06-28 15:45:16 -03:00
parent dbfae2d9bf
commit 7e17498bcf
1 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -46,12 +46,8 @@ export async function generateRequestParams(
} }
} }
export async function scrapWithFireEngine( export async function scrapWithFireEngine(
url: string, { url, waitFor = 0, screenshot = false, pageOptions = { parsePDF: true }, headers, options }:
waitFor: number = 0, { url: string, waitFor?: number, screenshot?: boolean, pageOptions?: { scrollXPaths?: string[], parsePDF?: boolean }, headers?: Record<string, string>, options?: any }
screenshot: boolean = false,
pageOptions: { scrollXPaths?: string[], parsePDF?: boolean } = { parsePDF: true },
headers?: Record<string, string>,
options?: any
): Promise<FireEngineResponse> { ): Promise<FireEngineResponse> {
try { try {
const reqParams = await generateRequestParams(url); const reqParams = await generateRequestParams(url);
@ -321,11 +317,13 @@ export async function scrapSingleUrl(
case "fire-engine": case "fire-engine":
if (process.env.FIRE_ENGINE_BETA_URL) { if (process.env.FIRE_ENGINE_BETA_URL) {
console.log(`Scraping ${url} with Fire Engine`); console.log(`Scraping ${url} with Fire Engine`);
const response = await scrapWithFireEngine( const response = await scrapWithFireEngine({
url, url,
pageOptions.waitFor, waitFor: pageOptions.waitFor,
pageOptions.screenshot, screenshot: pageOptions.screenshot,
pageOptions.headers pageOptions: pageOptions,
headers: pageOptions.headers
}
); );
scraperResponse.text = response.html; scraperResponse.text = response.html;
scraperResponse.screenshot = response.screenshot; scraperResponse.screenshot = response.screenshot;
@ -377,7 +375,7 @@ export async function scrapSingleUrl(
if (customScraperResult){ if (customScraperResult){
switch (customScraperResult.scraper) { switch (customScraperResult.scraper) {
case "fire-engine": case "fire-engine":
customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions) customScrapedContent = await scrapWithFireEngine({url: customScraperResult.url, waitFor: customScraperResult.waitAfterLoad, screenshot: false, pageOptions: customScraperResult.pageOptions})
if (screenshot) { if (screenshot) {
customScrapedContent.screenshot = screenshot; customScrapedContent.screenshot = screenshot;
} }