AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 20:16:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed file blocking process

Browse Source
This commit is contained in:
rafaelsideguide 2024-10-14 12:18:26 -03:00
parent a6888ce17b
commit 2bf7b433e2
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts
View File

@ -1,5 +1,5 @@
import axios, { AxiosResponse } from "axios"; import axios, { AxiosResponse } from "axios";
import fs from "fs"; import fs from "fs/promises";
import { createReadStream, createWriteStream } from "node:fs"; import { createReadStream, createWriteStream } from "node:fs";
import FormData from "form-data"; import FormData from "form-data";
import dotenv from "dotenv"; import dotenv from "dotenv";
@ -15,7 +15,7 @@ export async function fetchAndProcessPdf(url: string, parsePDF: boolean): Promis
try { try {
const { tempFilePath, pageStatusCode, pageError } = await downloadPdf(url); const { tempFilePath, pageStatusCode, pageError } = await downloadPdf(url);
const content = await processPdfToText(tempFilePath, parsePDF); const content = await processPdfToText(tempFilePath, parsePDF);
fs.unlinkSync(tempFilePath); // Clean up the temporary file await fs.unlink(tempFilePath); // Clean up the temporary file
return { content, pageStatusCode, pageError }; return { content, pageStatusCode, pageError };
} catch (error) { } catch (error) {
Logger.error(`Failed to fetch and process PDF: ${error.message}`); Logger.error(`Failed to fetch and process PDF: ${error.message}`);
@ -120,7 +120,7 @@ export async function processPdfToText(filePath: string, parsePDF: boolean): Pro
} }
} else { } else {
try { try {
content = fs.readFileSync(filePath, "utf-8"); content = await fs.readFile(filePath, "utf-8");
} catch (error) { } catch (error) {
Logger.error(`Failed to read PDF file: ${error}`); Logger.error(`Failed to read PDF file: ${error}`);
content = ""; content = "";
@ -131,7 +131,7 @@ export async function processPdfToText(filePath: string, parsePDF: boolean): Pro
async function processPdf(file: string) { async function processPdf(file: string) {
try { try {
const fileContent = fs.readFileSync(file); const fileContent = await fs.readFile(file);
const data = await pdf(fileContent); const data = await pdf(fileContent);
return data.text; return data.text;
} catch (error) { } catch (error) {