mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-08-02 02:50:37 +08:00
fix(llm-extract): handle llm-extract if scrape failed
This commit is contained in:
parent
670d253a8c
commit
e4adbaa88e
@ -15,7 +15,7 @@ const defaultPrompt =
|
|||||||
function prepareOpenAIDoc(
|
function prepareOpenAIDoc(
|
||||||
document: Document,
|
document: Document,
|
||||||
mode: "markdown" | "raw-html"
|
mode: "markdown" | "raw-html"
|
||||||
): [OpenAI.Chat.Completions.ChatCompletionContentPart[], number] {
|
): [OpenAI.Chat.Completions.ChatCompletionContentPart[], number] | null {
|
||||||
|
|
||||||
let markdown = document.markdown;
|
let markdown = document.markdown;
|
||||||
|
|
||||||
@ -27,9 +27,10 @@ function prepareOpenAIDoc(
|
|||||||
|
|
||||||
// Check if the markdown content exists in the document
|
// Check if the markdown content exists in the document
|
||||||
if (!extractionTarget) {
|
if (!extractionTarget) {
|
||||||
throw new Error(
|
return null;
|
||||||
`${mode} content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai`
|
// throw new Error(
|
||||||
);
|
// `${mode} content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai`
|
||||||
|
// );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -64,7 +65,16 @@ export async function generateOpenAICompletions({
|
|||||||
mode: "markdown" | "raw-html";
|
mode: "markdown" | "raw-html";
|
||||||
}): Promise<Document> {
|
}): Promise<Document> {
|
||||||
const openai = client as OpenAI;
|
const openai = client as OpenAI;
|
||||||
const [content, numTokens] = prepareOpenAIDoc(document, mode);
|
const preparedDoc = prepareOpenAIDoc(document, mode);
|
||||||
|
|
||||||
|
if (preparedDoc === null) {
|
||||||
|
return {
|
||||||
|
...document,
|
||||||
|
warning: "LLM extraction was not performed since the document's content is empty or missing.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const [content, numTokens] = preparedDoc;
|
||||||
|
|
||||||
const completion = await openai.chat.completions.create({
|
const completion = await openai.chat.completions.create({
|
||||||
model,
|
model,
|
||||||
|
Loading…
x
Reference in New Issue
Block a user