AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-22 04:22:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update index.ts

Browse Source
This commit is contained in:
Nicolas 2024-12-27 20:59:18 -03:00
parent f9d55efba8
commit 1eca61bffb
1 changed files with 13 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
View File

@ -8,7 +8,7 @@ import escapeHtml from "escape-html";
import PdfParse from "pdf-parse";
import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
import { RemoveFeatureError, UnsupportedFileError } from "../../error";
import { stat, readFile, unlink } from "node:fs/promises";
import { readFile, unlink } from "node:fs/promises";
import path from "node:path";
type PDFProcessorResult = { html: string; markdown?: string };
@ -25,7 +25,6 @@ async function scrapePDFWithRunPodMU(
tempFilePath,
});
const result = await robustFetch({
url:
"https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",
@ -92,15 +91,14 @@ export async function scrapePDF(
const base64Content = (await readFile(tempFilePath)).toString("base64");
// Then, if output is too short, pass to RunPod MU
// First try RunPod MU if conditions are met
if (
// result.markdown && result.markdown.length < 500 &&
base64Content.length < MAX_FILE_SIZE &&
process.env.RUNPOD_MU_API_KEY &&
process.env.RUNPOD_MU_POD_ID
) {
try {
const muResult = await scrapePDFWithRunPodMU(
result = await scrapePDFWithRunPodMU(
{
...meta,
logger: meta.logger.child({
@ -111,24 +109,20 @@ export async function scrapePDF(
timeToRun,
base64Content,
);
result = muResult; // Use LlamaParse result if successful
} catch (error) {
if (error instanceof Error && error.message === "RunPod MU timed out") {
meta.logger.warn("RunPod MU timed out -- using parse-pdf result", {
error,
});
} else if (error instanceof RemoveFeatureError) {
if (error instanceof RemoveFeatureError) {
throw error;
} else {
meta.logger.warn(
"RunPod MU failed to parse PDF -- using parse-pdf result",
{ error },
);
Sentry.captureException(error);
}
meta.logger.warn(
"RunPod MU failed to parse PDF -- falling back to parse-pdf",
{ error },
);
Sentry.captureException(error);
}
} else {
// First, try parsing with PdfParse
}
// If RunPod MU failed or wasn't attempted, use PdfParse
if (!result) {
result = await scrapePDFWithParsePDF(
{
...meta,
@ -145,7 +139,6 @@ export async function scrapePDF(
return {
url: response.url,
statusCode: response.status,
html: result?.html ?? "",
markdown: result?.markdown ?? "",
};