AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-15 05:15:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/remove-base64-images

Browse Source
This commit is contained in:
Rafael Miller 2024-11-04 10:35:09 -03:00 committed by GitHub
commit 8297e5beef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v1/types.ts
View File

@ -116,6 +116,7 @@ export const scrapeOptions = z.object({
timeout: z.number().int().positive().finite().safe().default(30000), timeout: z.number().int().positive().finite().safe().default(30000),
waitFor: z.number().int().nonnegative().finite().safe().default(0), waitFor: z.number().int().nonnegative().finite().safe().default(0),
extract: extractOptions.optional(), extract: extractOptions.optional(),
mobile: z.boolean().default(false),
parsePDF: z.boolean().default(true), parsePDF: z.boolean().default(true),
actions: actionsSchema.optional(), actions: actionsSchema.optional(),
// New // New
@ -471,6 +472,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions {
geolocation: x.location ?? x.geolocation, geolocation: x.location ?? x.geolocation,
skipTlsVerification: x.skipTlsVerification, skipTlsVerification: x.skipTlsVerification,
removeBase64Images: x.removeBase64Images, removeBase64Images: x.removeBase64Images,
mobile: x.mobile,
}; };
} }

1
apps/api/src/lib/entities.ts
View File

@ -59,6 +59,7 @@ export type PageOptions = {
}; };
skipTlsVerification?: boolean; skipTlsVerification?: boolean;
removeBase64Images?: boolean; removeBase64Images?: boolean;
mobile?: boolean;
}; };
export type ExtractorOptions = { export type ExtractorOptions = {

1
apps/api/src/scraper/WebScraper/index.ts
View File

@ -596,6 +596,7 @@ export class WebScraperDataProvider {
geolocation: options.pageOptions?.geolocation ?? undefined, geolocation: options.pageOptions?.geolocation ?? undefined,
skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false, skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,
removeBase64Images: options.pageOptions?.removeBase64Images ?? true, removeBase64Images: options.pageOptions?.removeBase64Images ?? true,
mobile: options.pageOptions?.mobile ?? false,
}; };
this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };
this.replaceAllPathsWithAbsolutePaths = this.replaceAllPathsWithAbsolutePaths =

5
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -28,7 +28,7 @@ export async function scrapWithFireEngine({
waitFor = 0, waitFor = 0,
screenshot = false, screenshot = false,
fullPageScreenshot = false, fullPageScreenshot = false,
pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true }, pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true, mobile: false },
fireEngineOptions = {}, fireEngineOptions = {},
headers, headers,
options, options,
@ -40,7 +40,7 @@ export async function scrapWithFireEngine({
waitFor?: number; waitFor?: number;
screenshot?: boolean; screenshot?: boolean;
fullPageScreenshot?: boolean; fullPageScreenshot?: boolean;
pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean }; pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean, mobile?: boolean };
fireEngineOptions?: FireEngineOptions; fireEngineOptions?: FireEngineOptions;
headers?: Record<string, string>; headers?: Record<string, string>;
options?: any; options?: any;
@ -115,6 +115,7 @@ export async function scrapWithFireEngine({
priority, priority,
engine, engine,
instantReturn: true, instantReturn: true,
mobile: pageOptions?.mobile ?? false,
...fireEngineOptionsParam, ...fireEngineOptionsParam,
atsv: pageOptions?.atsv ?? false, atsv: pageOptions?.atsv ?? false,
scrollXPaths: pageOptions?.scrollXPaths ?? [], scrollXPaths: pageOptions?.scrollXPaths ?? [],

1
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -161,6 +161,7 @@ export async function scrapSingleUrl(
geolocation: pageOptions.geolocation ?? undefined, geolocation: pageOptions.geolocation ?? undefined,
skipTlsVerification: pageOptions.skipTlsVerification ?? false, skipTlsVerification: pageOptions.skipTlsVerification ?? false,
removeBase64Images: pageOptions.removeBase64Images ?? true, removeBase64Images: pageOptions.removeBase64Images ?? true,
mobile: pageOptions.mobile ?? false,
} }
if (extractorOptions) { if (extractorOptions) {