AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 19:36:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update fetch.ts

Browse Source
This commit is contained in:
Nicolas 2024-10-03 18:56:51 -03:00
parent 668ff3c71b
commit dba96998e3
1 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
apps/api/src/scraper/WebScraper/scrapers/fetch.ts
View File

@ -35,7 +35,9 @@ export async function scrapWithFetch(
}); });
if (response.status !== 200) { if (response.status !== 200) {
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`); Logger.debug(
`⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`
);
logParams.error_message = response.statusText; logParams.error_message = response.statusText;
logParams.response_code = response.status; logParams.response_code = response.status;
return { return {
@ -48,7 +50,10 @@ export async function scrapWithFetch(
const contentType = response.headers["content-type"]; const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) { if (contentType && contentType.includes("application/pdf")) {
logParams.success = true; logParams.success = true;
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF); const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
url,
pageOptions?.parsePDF
);
logParams.response_code = pageStatusCode; logParams.response_code = pageStatusCode;
logParams.error_message = pageError; logParams.error_message = pageError;
return { content, pageStatusCode: response.status, pageError }; return { content, pageStatusCode: response.status, pageError };
@ -57,7 +62,11 @@ export async function scrapWithFetch(
logParams.success = true; logParams.success = true;
logParams.html = text; logParams.html = text;
logParams.response_code = response.status; logParams.response_code = response.status;
return { content: text, pageStatusCode: response.status, pageError: null }; return {
content: text,
pageStatusCode: response.status,
pageError: null,
};
} }
} catch (error) { } catch (error) {
if (error.code === "ECONNABORTED") { if (error.code === "ECONNABORTED") {
@ -67,7 +76,11 @@ export async function scrapWithFetch(
logParams.error_message = error.message || error; logParams.error_message = error.message || error;
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`); Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);
} }
return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message }; return {
content: "",
pageStatusCode: error.response?.status ?? null,
pageError: logParams.error_message,
};
} finally { } finally {
const endTime = Date.now(); const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000; logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;