diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts index e4c3802f..d6bd5157 100644 --- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts +++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts @@ -123,10 +123,11 @@ export async function robustFetch< return null as Output; } + const resp = await request.text(); response = { status: request.status, headers: request.headers, - body: await request.text(), // NOTE: can this throw an exception? + body: resp, // NOTE: can this throw an exception? }; } else { if (ignoreResponse === true) { @@ -171,7 +172,7 @@ export async function robustFetch< if (tryCount > 1) { logger.debug( "Request sent failure status, trying " + (tryCount - 1) + " more times", - { params, response, requestId }, + { params: { ...params, logger: undefined }, response: { status: response.status, body: response.body }, requestId }, ); if (tryCooldown !== undefined) { await new Promise((resolve) => @@ -186,14 +187,14 @@ export async function robustFetch< }); } else { logger.debug("Request sent failure status", { - params, - response, + params: { ...params, logger: undefined }, + response: { status: response.status, body: response.body }, requestId, }); throw new Error("Request sent failure status", { cause: { - params, - response, + params: { ...params, logger: undefined }, + response: { status: response.status, body: response.body }, requestId, }, }); @@ -217,13 +218,13 @@ export async function robustFetch< data = JSON.parse(response.body); } catch (error) { logger.debug("Request sent malformed JSON", { - params, - response, + params: { ...params, logger: undefined }, + response: { status: response.status, body: response.body }, requestId, }); throw new Error("Request sent malformed JSON", { cause: { - params, + params: { ...params, logger: undefined }, response, requestId, }, @@ -236,15 +237,15 @@ export async function robustFetch< } catch (error) { if (error instanceof ZodError) { logger.debug("Response does not match provided schema", { - params, - response, + params: { ...params, logger: undefined }, + response: { status: response.status, body: response.body }, requestId, error, schema, }); throw new Error("Response does not match provided schema", { cause: { - params, + params: { ...params, logger: undefined }, response, requestId, error, @@ -253,15 +254,15 @@ export async function robustFetch< }); } else { logger.debug("Parsing response with provided schema failed", { - params, - response, + params: { ...params, logger: undefined }, + response: { status: response.status, body: response.body }, requestId, error, schema, }); throw new Error("Parsing response with provided schema failed", { cause: { - params, + params: { ...params, logger: undefined }, response, requestId, error,