AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-17 10:05:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-28 17:17:17 +02:00
parent 6256d7a0fe
commit 3e1e069e01
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/scraper/scrapeURL/engines/index.ts
View File

@ -439,10 +439,19 @@ export function buildFallbackList(meta: Meta): {
) )
&& ( && (
meta.options.actions === undefined meta.options.actions === undefined
|| Object.keys(meta.options.actions).length === 0 || meta.options.actions.length === 0
) )
&& meta.options.proxy !== "stealth"; && meta.options.proxy !== "stealth";
meta.logger.warn("shouldUseIndex", {
shouldUseIndex,
formatsNoChangeTracking: !meta.options.formats.includes("changeTracking"),
maxAge: meta.options.maxAge !== 0,
headers: meta.options.headers === undefined || Object.keys(meta.options.headers).length === 0,
actions: meta.options.actions === undefined || meta.options.actions.length === 0,
proxy: meta.options.proxy !== "stealth",
});
if (!shouldUseIndex) { if (!shouldUseIndex) {
const indexIndex = _engines.indexOf("index"); const indexIndex = _engines.indexOf("index");
if (indexIndex !== -1) { if (indexIndex !== -1) {

11
apps/api/src/scraper/scrapeURL/engines/index/index.ts
View File

@ -15,6 +15,13 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
|| Object.keys(meta.options.headers).length === 0 || Object.keys(meta.options.headers).length === 0
); );
meta.logger.warn("shouldCache", {
shouldCache,
winnerEngine: meta.winnerEngine !== "cache" && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents",
featureFlags: !meta.featureFlags.has("actions"),
headers: meta.options.headers === undefined || Object.keys(meta.options.headers).length === 0,
});
if (!shouldCache) { if (!shouldCache) {
return document; return document;
} }
@ -57,8 +64,8 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
has_screenshot_fullscreen: document.screenshot !== undefined && meta.featureFlags.has("screenshot@fullScreen"), has_screenshot_fullscreen: document.screenshot !== undefined && meta.featureFlags.has("screenshot@fullScreen"),
is_mobile: meta.options.mobile, is_mobile: meta.options.mobile,
block_ads: meta.options.blockAds, block_ads: meta.options.blockAds,
location: meta.options.location?.country ?? null, location_country: meta.options.location?.country ?? null,
languages: meta.options.location?.languages ?? null, location_languages: meta.options.location?.languages ?? null,
status: document.metadata.statusCode, status: document.metadata.statusCode,
}); });