AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-04 11:24:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(robustFetch): selective logging (#1588)

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-22 22:23:49 +02:00 committed by GitHub
parent 749d89a551
commit cc2c968425
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -6,6 +6,7 @@ import { TimeoutSignal } from "../../../controllers/v1/types";
import { fireEngineURL } from "../engines/fire-engine/scrape"; import { fireEngineURL } from "../engines/fire-engine/scrape";
import { fetch, RequestInit, Response, FormData, Agent } from "undici"; import { fetch, RequestInit, Response, FormData, Agent } from "undici";
import { cacheableLookup } from "./cacheableLookup"; import { cacheableLookup } from "./cacheableLookup";
import { log } from "console";
export type RobustFetchParams<Schema extends z.Schema<any>> = { export type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string; url: string;
@ -58,6 +59,19 @@ export async function robustFetch<
abort, abort,
}; };
// omit pdf file content from logs
const logParams = {
...params,
body: body?.input ? {
...body,
input: {
...body.input,
file_content: undefined,
},
} : body,
logger: undefined,
}
let response: { let response: {
status: number; status: number;
headers: Headers; headers: Headers;
@ -105,7 +119,7 @@ export async function robustFetch<
if (tryCount > 1) { if (tryCount > 1) {
logger.debug( logger.debug(
"Request failed, trying " + (tryCount - 1) + " more times", "Request failed, trying " + (tryCount - 1) + " more times",
{ params, error, requestId }, { params: logParams, error, requestId },
); );
return await robustFetch({ return await robustFetch({
...params, ...params,
@ -114,7 +128,7 @@ export async function robustFetch<
mock, mock,
}); });
} else { } else {
logger.debug("Request failed", { params, error, requestId }); logger.debug("Request failed", { params: logParams, error, requestId });
throw new Error("Request failed", { throw new Error("Request failed", {
cause: { cause: {
params, params,
@ -181,7 +195,7 @@ export async function robustFetch<
if (tryCount > 1) { if (tryCount > 1) {
logger.debug( logger.debug(
"Request sent failure status, trying " + (tryCount - 1) + " more times", "Request sent failure status, trying " + (tryCount - 1) + " more times",
{ params: { ...params, logger: undefined }, response: { status: response.status, body: response.body }, requestId }, { params: logParams, response: { status: response.status, body: response.body }, requestId },
); );
if (tryCooldown !== undefined) { if (tryCooldown !== undefined) {
await new Promise((resolve) => await new Promise((resolve) =>
@ -196,13 +210,13 @@ export async function robustFetch<
}); });
} else { } else {
logger.debug("Request sent failure status", { logger.debug("Request sent failure status", {
params: { ...params, logger: undefined }, params: logParams,
response: { status: response.status, body: response.body }, response: { status: response.status, body: response.body },
requestId, requestId,
}); });
throw new Error("Request sent failure status", { throw new Error("Request sent failure status", {
cause: { cause: {
params: { ...params, logger: undefined }, params: logParams,
response: { status: response.status, body: response.body }, response: { status: response.status, body: response.body },
requestId, requestId,
}, },
@ -227,13 +241,13 @@ export async function robustFetch<
data = JSON.parse(response.body); data = JSON.parse(response.body);
} catch (error) { } catch (error) {
logger.debug("Request sent malformed JSON", { logger.debug("Request sent malformed JSON", {
params: { ...params, logger: undefined }, params: logParams,
response: { status: response.status, body: response.body }, response: { status: response.status, body: response.body },
requestId, requestId,
}); });
throw new Error("Request sent malformed JSON", { throw new Error("Request sent malformed JSON", {
cause: { cause: {
params: { ...params, logger: undefined }, params: logParams,
response, response,
requestId, requestId,
}, },
@ -246,7 +260,7 @@ export async function robustFetch<
} catch (error) { } catch (error) {
if (error instanceof ZodError) { if (error instanceof ZodError) {
logger.debug("Response does not match provided schema", { logger.debug("Response does not match provided schema", {
params: { ...params, logger: undefined }, params: logParams,
response: { status: response.status, body: response.body }, response: { status: response.status, body: response.body },
requestId, requestId,
error, error,
@ -254,7 +268,7 @@ export async function robustFetch<
}); });
throw new Error("Response does not match provided schema", { throw new Error("Response does not match provided schema", {
cause: { cause: {
params: { ...params, logger: undefined }, params: logParams,
response, response,
requestId, requestId,
error, error,
@ -263,7 +277,7 @@ export async function robustFetch<
}); });
} else { } else {
logger.debug("Parsing response with provided schema failed", { logger.debug("Parsing response with provided schema failed", {
params: { ...params, logger: undefined }, params: logParams,
response: { status: response.status, body: response.body }, response: { status: response.status, body: response.body },
requestId, requestId,
error, error,
@ -271,7 +285,7 @@ export async function robustFetch<
}); });
throw new Error("Parsing response with provided schema failed", { throw new Error("Parsing response with provided schema failed", {
cause: { cause: {
params: { ...params, logger: undefined }, params: logParams,
response, response,
requestId, requestId,
error, error,