AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 15:29:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor robustFetch logging to exclude sensitive parameters and improve error handling.

Browse Source
This commit is contained in:
rafaelmmiller 2025-04-15 17:15:01 -07:00
parent a840db9ef3
commit 252a9ccc89
1 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -123,10 +123,11 @@ export async function robustFetch<
return null as Output; return null as Output;
} }
const resp = await request.text();
response = { response = {
status: request.status, status: request.status,
headers: request.headers, headers: request.headers,
body: await request.text(), // NOTE: can this throw an exception? body: resp, // NOTE: can this throw an exception?
}; };
} else { } else {
if (ignoreResponse === true) { if (ignoreResponse === true) {
@ -171,7 +172,7 @@ export async function robustFetch<
if (tryCount > 1) { if (tryCount > 1) {
logger.debug( logger.debug(
"Request sent failure status, trying " + (tryCount - 1) + " more times", "Request sent failure status, trying " + (tryCount - 1) + " more times",
{ params, response, requestId }, { params: { ...params, logger: undefined }, response: { status: response.status, body: response.body }, requestId },
); );
if (tryCooldown !== undefined) { if (tryCooldown !== undefined) {
await new Promise((resolve) => await new Promise((resolve) =>
@ -186,14 +187,14 @@ export async function robustFetch<
}); });
} else { } else {
logger.debug("Request sent failure status", { logger.debug("Request sent failure status", {
params, params: { ...params, logger: undefined },
response, response: { status: response.status, body: response.body },
requestId, requestId,
}); });
throw new Error("Request sent failure status", { throw new Error("Request sent failure status", {
cause: { cause: {
params, params: { ...params, logger: undefined },
response, response: { status: response.status, body: response.body },
requestId, requestId,
}, },
}); });
@ -217,13 +218,13 @@ export async function robustFetch<
data = JSON.parse(response.body); data = JSON.parse(response.body);
} catch (error) { } catch (error) {
logger.debug("Request sent malformed JSON", { logger.debug("Request sent malformed JSON", {
params, params: { ...params, logger: undefined },
response, response: { status: response.status, body: response.body },
requestId, requestId,
}); });
throw new Error("Request sent malformed JSON", { throw new Error("Request sent malformed JSON", {
cause: { cause: {
params, params: { ...params, logger: undefined },
response, response,
requestId, requestId,
}, },
@ -236,15 +237,15 @@ export async function robustFetch<
} catch (error) { } catch (error) {
if (error instanceof ZodError) { if (error instanceof ZodError) {
logger.debug("Response does not match provided schema", { logger.debug("Response does not match provided schema", {
params, params: { ...params, logger: undefined },
response, response: { status: response.status, body: response.body },
requestId, requestId,
error, error,
schema, schema,
}); });
throw new Error("Response does not match provided schema", { throw new Error("Response does not match provided schema", {
cause: { cause: {
params, params: { ...params, logger: undefined },
response, response,
requestId, requestId,
error, error,
@ -253,15 +254,15 @@ export async function robustFetch<
}); });
} else { } else {
logger.debug("Parsing response with provided schema failed", { logger.debug("Parsing response with provided schema failed", {
params, params: { ...params, logger: undefined },
response, response: { status: response.status, body: response.body },
requestId, requestId,
error, error,
schema, schema,
}); });
throw new Error("Parsing response with provided schema failed", { throw new Error("Parsing response with provided schema failed", {
cause: { cause: {
params, params: { ...params, logger: undefined },
response, response,
requestId, requestId,
error, error,