AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 11:30:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update fetch.ts

Browse Source
This commit is contained in:
Nicolas 2024-10-03 18:56:51 -03:00
parent 668ff3c71b
commit dba96998e3
1 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
apps/api/src/scraper/WebScraper/scrapers/fetch.ts
View File

@ -35,7 +35,9 @@ export async function scrapWithFetch(
});
if (response.status !== 200) {
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`);
Logger.debug(
`⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`
);
logParams.error_message = response.statusText;
logParams.response_code = response.status;
return {
@ -48,7 +50,10 @@ export async function scrapWithFetch(
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
logParams.success = true;
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
url,
pageOptions?.parsePDF
);
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
return { content, pageStatusCode: response.status, pageError };
@ -57,7 +62,11 @@ export async function scrapWithFetch(
logParams.success = true;
logParams.html = text;
logParams.response_code = response.status;
return { content: text, pageStatusCode: response.status, pageError: null };
return {
content: text,
pageStatusCode: response.status,
pageError: null,
};
}
} catch (error) {
if (error.code === "ECONNABORTED") {
@ -67,7 +76,11 @@ export async function scrapWithFetch(
logParams.error_message = error.message || error;
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);
}
return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message };
return {
content: "",
pageStatusCode: error.response?.status ?? null,
pageError: logParams.error_message,
};
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;