AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 02:28:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor robustFetch logging to exclude sensitive parameters and improve error handling.

Browse Source
This commit is contained in:
rafaelmmiller 2025-04-15 17:15:01 -07:00
parent a840db9ef3
commit 252a9ccc89
1 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -123,10 +123,11 @@ export async function robustFetch<
return null as Output;
}
const resp = await request.text();
response = {
status: request.status,
headers: request.headers,
body: await request.text(), // NOTE: can this throw an exception?
body: resp, // NOTE: can this throw an exception?
};
} else {
if (ignoreResponse === true) {
@ -171,7 +172,7 @@ export async function robustFetch<
if (tryCount > 1) {
logger.debug(
"Request sent failure status, trying " + (tryCount - 1) + " more times",
{ params, response, requestId },
{ params: { ...params, logger: undefined }, response: { status: response.status, body: response.body }, requestId },
);
if (tryCooldown !== undefined) {
await new Promise((resolve) =>
@ -186,14 +187,14 @@ export async function robustFetch<
});
} else {
logger.debug("Request sent failure status", {
params,
response,
params: { ...params, logger: undefined },
response: { status: response.status, body: response.body },
requestId,
});
throw new Error("Request sent failure status", {
cause: {
params,
response,
params: { ...params, logger: undefined },
response: { status: response.status, body: response.body },
requestId,
},
});
@ -217,13 +218,13 @@ export async function robustFetch<
data = JSON.parse(response.body);
} catch (error) {
logger.debug("Request sent malformed JSON", {
params,
response,
params: { ...params, logger: undefined },
response: { status: response.status, body: response.body },
requestId,
});
throw new Error("Request sent malformed JSON", {
cause: {
params,
params: { ...params, logger: undefined },
response,
requestId,
},
@ -236,15 +237,15 @@ export async function robustFetch<
} catch (error) {
if (error instanceof ZodError) {
logger.debug("Response does not match provided schema", {
params,
response,
params: { ...params, logger: undefined },
response: { status: response.status, body: response.body },
requestId,
error,
schema,
});
throw new Error("Response does not match provided schema", {
cause: {
params,
params: { ...params, logger: undefined },
response,
requestId,
error,
@ -253,15 +254,15 @@ export async function robustFetch<
});
} else {
logger.debug("Parsing response with provided schema failed", {
params,
response,
params: { ...params, logger: undefined },
response: { status: response.status, body: response.body },
requestId,
error,
schema,
});
throw new Error("Parsing response with provided schema failed", {
cause: {
params,
params: { ...params, logger: undefined },
response,
requestId,
error,