AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 04:36:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add scrapeOptions.fastMode

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-13 22:30:57 +01:00
parent 588f747ee8
commit 842b522b44
3 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/controllers/v1/types.ts
View File

@ -182,6 +182,7 @@ export const scrapeOptions = z
.optional(),
skipTlsVerification: z.boolean().default(false),
removeBase64Images: z.boolean().default(true),
fastMode: z.boolean().default(false),
})
.strict(strictMessage);
@ -685,11 +686,11 @@ export function fromLegacyScrapeOptions(
}
: undefined,
mobile: pageOptions.mobile,
fastMode: pageOptions.useFastMode,
}),
internalOptions: {
atsv: pageOptions.atsv,
v0DisableJsDom: pageOptions.disableJsDom,
v0UseFastMode: pageOptions.useFastMode,
},
// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
};

2
apps/api/src/lib/cache.ts
View File

@ -21,7 +21,7 @@ export function cacheKey(
if (
internalOptions.v0CrawlOnlyUrls ||
internalOptions.forceEngine ||
internalOptions.v0UseFastMode ||
scrapeOptions.fastMode ||
internalOptions.atsv ||
(scrapeOptions.actions && scrapeOptions.actions.length > 0)
) {

3
apps/api/src/scraper/scrapeURL/index.ts
View File

@ -86,7 +86,7 @@ function buildFeatureFlags(
flags.add("skipTlsVerification");
}
if (internalOptions.v0UseFastMode) {
if (options.fastMode) {
flags.add("useFastMode");
}
@ -148,7 +148,6 @@ export type InternalOptions = {
atsv?: boolean; // anti-bot solver, beta
v0CrawlOnlyUrls?: boolean;
v0UseFastMode?: boolean;
v0DisableJsDom?: boolean;
disableSmartWaitCache?: boolean; // Passed along to fire-engine