AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 00:10:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

minor fixes

Browse Source
This commit is contained in:
Gergő Móricz 2024-09-18 21:34:09 +02:00
parent 20d1855ad5
commit 712ca31615
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/index.ts
View File

@ -591,7 +591,8 @@ export class WebScraperDataProvider {
screenshot: options.pageOptions?.screenshot ?? false, screenshot: options.pageOptions?.screenshot ?? false,
useFastMode: options.pageOptions?.useFastMode ?? false, useFastMode: options.pageOptions?.useFastMode ?? false,
disableJsDom: options.pageOptions?.disableJsDom ?? false, disableJsDom: options.pageOptions?.disableJsDom ?? false,
atsv: options.pageOptions?.atsv ?? false atsv: options.pageOptions?.atsv ?? false,
actions: options.pageOptions?.actions ?? undefined,
}; };
this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };
this.replaceAllPathsWithAbsolutePaths = this.replaceAllPathsWithAbsolutePaths =

4
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -116,7 +116,7 @@ export async function scrapWithFireEngine({
); );
}); });
const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => x.milliseconds + a, 0); const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => (x as { type: "wait"; milliseconds: number; }).milliseconds + a, 0);
let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);
while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal) { while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal) {
@ -141,7 +141,7 @@ export async function scrapWithFireEngine({
if (checkStatusResponse.status !== 200 || checkStatusResponse.data.error) { if (checkStatusResponse.status !== 200 || checkStatusResponse.data.error) {
Logger.debug( Logger.debug(
`⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${checkStatusResponse.status}` `⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${checkStatusResponse.status}\t ${checkStatusResponse.data.error}`
); );
logParams.error_message = checkStatusResponse.data?.pageError ?? checkStatusResponse.data?.error; logParams.error_message = checkStatusResponse.data?.pageError ?? checkStatusResponse.data?.error;

3
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -153,7 +153,8 @@ export async function scrapSingleUrl(
onlyIncludeTags: pageOptions.onlyIncludeTags ?? [], onlyIncludeTags: pageOptions.onlyIncludeTags ?? [],
useFastMode: pageOptions.useFastMode ?? false, useFastMode: pageOptions.useFastMode ?? false,
disableJsDom: pageOptions.disableJsDom ?? false, disableJsDom: pageOptions.disableJsDom ?? false,
atsv: pageOptions.atsv ?? false atsv: pageOptions.atsv ?? false,
actions: pageOptions.actions ?? undefined,
} }
if (extractorOptions) { if (extractorOptions) {