AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 21:29:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve logging

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-15 23:52:20 -07:00
parent 0935ec210e
commit 2193bee133
3 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/lib/extract/completions/singleAnswer.ts
View File

@ -23,7 +23,7 @@ export async function singleAnswerCompletion({
prompt: string;
systemPrompt: string;
useAgent: boolean;
extractId?: string;
extractId: string;
}): Promise<{
extract: any;
tokenUsage: TokenUsage;
@ -35,7 +35,11 @@ export async function singleAnswerCompletion({
}> {
const docsPrompt = `Today is: ` + new Date().toISOString() + `.\n` + prompt;
const generationOptions: GenerateCompletionsOptions = {
logger: logger.child({ module: "extract", method: "generateCompletions" }),
logger: logger.child({
module: "extract",
method: "generateCompletions",
extractId,
}),
options: {
mode: "llm",
systemPrompt:

18
apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
View File

@ -211,7 +211,7 @@ export async function extractData({
if (!schema && extractOptions.options.prompt) {
logger.info("Generating schema from prompt");
const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt);
const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt, logger);
otherCallCount++;
otherCost += genRes.cost;
schema = genRes.extract;
@ -249,7 +249,7 @@ export async function extractData({
} catch (error) {
logger.error(
"failed during extractSmartScrape.ts:generateCompletions",
error,
{ error },
);
// console.log("failed during extractSmartScrape.ts:generateCompletions", error);
}
@ -260,16 +260,12 @@ export async function extractData({
// console.log("smartscrape_reasoning", extract?.smartscrape_reasoning);
// console.log("smartscrape_prompt", extract?.smartscrape_prompt);
try {
console.log("=========================================");
console.log(
"useAgent:",
logger.info("Smart schema resolved", {
useAgent,
"shouldUseSmartscrape:",
extract?.shouldUseSmartscrape,
);
console.log("url:", urls);
console.log("prompt:", extract?.smartscrape_prompt);
console.log("=========================================");
shouldUseSmartscrape: extract?.shouldUseSmartscrape,
url: urls,
prompt: extract?.smartscrape_prompt,
})
if (useAgent && extract?.shouldUseSmartscrape) {
let smartscrapeResults: SmartScrapeResult[];

1
apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
View File

@ -761,6 +761,7 @@ export function removeDefaultProperty(schema: any): any {
export async function generateSchemaFromPrompt(
prompt: string,
logger: Logger,
): Promise<{ extract: any; cost: number }> {
const model = getModel("gpt-4o", "openai");
const retryModel = getModel("gpt-4o-mini", "openai");