AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-01 07:12:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(llm-extract): handle llm-extract if scrape failed

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-22 14:12:52 +02:00
parent 670d253a8c
commit e4adbaa88e
1 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/lib/LLM-extraction/models.ts
View File

@ -15,7 +15,7 @@ const defaultPrompt =
function prepareOpenAIDoc(
document: Document,
mode: "markdown" | "raw-html"
): [OpenAI.Chat.Completions.ChatCompletionContentPart[], number] {
): [OpenAI.Chat.Completions.ChatCompletionContentPart[], number] | null {
let markdown = document.markdown;
@ -27,9 +27,10 @@ function prepareOpenAIDoc(
// Check if the markdown content exists in the document
if (!extractionTarget) {
throw new Error(
`${mode} content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai`
);
return null;
// throw new Error(
// `${mode} content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai`
// );
}
@ -64,7 +65,16 @@ export async function generateOpenAICompletions({
mode: "markdown" | "raw-html";
}): Promise<Document> {
const openai = client as OpenAI;
const [content, numTokens] = prepareOpenAIDoc(document, mode);
const preparedDoc = prepareOpenAIDoc(document, mode);
if (preparedDoc === null) {
return {
...document,
warning: "LLM extraction was not performed since the document's content is empty or missing.",
};
}
const [content, numTokens] = preparedDoc;
const completion = await openai.chat.completions.create({
model,