AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 22:25:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add scrapeOptions.fastMode

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-13 22:30:57 +01:00
parent 588f747ee8
commit 842b522b44
3 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/controllers/v1/types.ts
View File

@ -182,6 +182,7 @@ export const scrapeOptions = z
.optional(), .optional(),
skipTlsVerification: z.boolean().default(false), skipTlsVerification: z.boolean().default(false),
removeBase64Images: z.boolean().default(true), removeBase64Images: z.boolean().default(true),
fastMode: z.boolean().default(false),
}) })
.strict(strictMessage); .strict(strictMessage);
@ -685,11 +686,11 @@ export function fromLegacyScrapeOptions(
} }
: undefined, : undefined,
mobile: pageOptions.mobile, mobile: pageOptions.mobile,
fastMode: pageOptions.useFastMode,
}), }),
internalOptions: { internalOptions: {
atsv: pageOptions.atsv, atsv: pageOptions.atsv,
v0DisableJsDom: pageOptions.disableJsDom, v0DisableJsDom: pageOptions.disableJsDom,
v0UseFastMode: pageOptions.useFastMode,
}, },
// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks // TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
}; };

2
apps/api/src/lib/cache.ts
View File

@ -21,7 +21,7 @@ export function cacheKey(
if ( if (
internalOptions.v0CrawlOnlyUrls || internalOptions.v0CrawlOnlyUrls ||
internalOptions.forceEngine || internalOptions.forceEngine ||
internalOptions.v0UseFastMode || scrapeOptions.fastMode ||
internalOptions.atsv || internalOptions.atsv ||
(scrapeOptions.actions && scrapeOptions.actions.length > 0) (scrapeOptions.actions && scrapeOptions.actions.length > 0)
) { ) {

3
apps/api/src/scraper/scrapeURL/index.ts
View File

@ -86,7 +86,7 @@ function buildFeatureFlags(
flags.add("skipTlsVerification"); flags.add("skipTlsVerification");
} }
if (internalOptions.v0UseFastMode) { if (options.fastMode) {
flags.add("useFastMode"); flags.add("useFastMode");
} }
@ -148,7 +148,6 @@ export type InternalOptions = {
atsv?: boolean; // anti-bot solver, beta atsv?: boolean; // anti-bot solver, beta
v0CrawlOnlyUrls?: boolean; v0CrawlOnlyUrls?: boolean;
v0UseFastMode?: boolean;
v0DisableJsDom?: boolean; v0DisableJsDom?: boolean;
disableSmartWaitCache?: boolean; // Passed along to fire-engine disableSmartWaitCache?: boolean; // Passed along to fire-engine