AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 04:18:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve logging

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-15 23:52:20 -07:00
parent 0935ec210e
commit 2193bee133
3 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/lib/extract/completions/singleAnswer.ts
View File

@ -23,7 +23,7 @@ export async function singleAnswerCompletion({
prompt: string; prompt: string;
systemPrompt: string; systemPrompt: string;
useAgent: boolean; useAgent: boolean;
extractId?: string; extractId: string;
}): Promise<{ }): Promise<{
extract: any; extract: any;
tokenUsage: TokenUsage; tokenUsage: TokenUsage;
@ -35,7 +35,11 @@ export async function singleAnswerCompletion({
}> { }> {
const docsPrompt = `Today is: ` + new Date().toISOString() + `.\n` + prompt; const docsPrompt = `Today is: ` + new Date().toISOString() + `.\n` + prompt;
const generationOptions: GenerateCompletionsOptions = { const generationOptions: GenerateCompletionsOptions = {
logger: logger.child({ module: "extract", method: "generateCompletions" }), logger: logger.child({
module: "extract",
method: "generateCompletions",
extractId,
}),
options: { options: {
mode: "llm", mode: "llm",
systemPrompt: systemPrompt:

18
apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
View File

@ -211,7 +211,7 @@ export async function extractData({
if (!schema && extractOptions.options.prompt) { if (!schema && extractOptions.options.prompt) {
logger.info("Generating schema from prompt"); logger.info("Generating schema from prompt");
const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt); const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt, logger);
otherCallCount++; otherCallCount++;
otherCost += genRes.cost; otherCost += genRes.cost;
schema = genRes.extract; schema = genRes.extract;
@ -249,7 +249,7 @@ export async function extractData({
} catch (error) { } catch (error) {
logger.error( logger.error(
"failed during extractSmartScrape.ts:generateCompletions", "failed during extractSmartScrape.ts:generateCompletions",
error, { error },
); );
// console.log("failed during extractSmartScrape.ts:generateCompletions", error); // console.log("failed during extractSmartScrape.ts:generateCompletions", error);
} }
@ -260,16 +260,12 @@ export async function extractData({
// console.log("smartscrape_reasoning", extract?.smartscrape_reasoning); // console.log("smartscrape_reasoning", extract?.smartscrape_reasoning);
// console.log("smartscrape_prompt", extract?.smartscrape_prompt); // console.log("smartscrape_prompt", extract?.smartscrape_prompt);
try { try {
console.log("========================================="); logger.info("Smart schema resolved", {
console.log(
"useAgent:",
useAgent, useAgent,
"shouldUseSmartscrape:", shouldUseSmartscrape: extract?.shouldUseSmartscrape,
extract?.shouldUseSmartscrape, url: urls,
); prompt: extract?.smartscrape_prompt,
console.log("url:", urls); })
console.log("prompt:", extract?.smartscrape_prompt);
console.log("=========================================");
if (useAgent && extract?.shouldUseSmartscrape) { if (useAgent && extract?.shouldUseSmartscrape) {
let smartscrapeResults: SmartScrapeResult[]; let smartscrapeResults: SmartScrapeResult[];

1
apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
View File

@ -761,6 +761,7 @@ export function removeDefaultProperty(schema: any): any {
export async function generateSchemaFromPrompt( export async function generateSchemaFromPrompt(
prompt: string, prompt: string,
logger: Logger,
): Promise<{ extract: any; cost: number }> { ): Promise<{ extract: any; cost: number }> {
const model = getModel("gpt-4o", "openai"); const model = getModel("gpt-4o", "openai");
const retryModel = getModel("gpt-4o-mini", "openai"); const retryModel = getModel("gpt-4o-mini", "openai");