diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts index 45c4d47b..0fda40db 100644 --- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts +++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts @@ -243,18 +243,18 @@ export async function extractData({ }), ); } - console.log("smartscrapeResults", smartscrapeResults); + // console.log("smartscrapeResults", smartscrapeResults); const scrapedPages = smartscrapeResults.map( (result) => result.scrapedPages, ); - console.log("scrapedPages", scrapedPages); + // console.log("scrapedPages", scrapedPages); const htmls = scrapedPages.flat().map((page) => page.html); - console.log("htmls", htmls); + // console.log("htmls", htmls); const markdowns = await Promise.all( htmls.map(async (html) => await parseMarkdown(html)), ); - console.log("markdowns", markdowns); + // console.log("markdowns", markdowns); extractedData = await Promise.all( markdowns.map(async (markdown) => { const newExtractOptions = { diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts index f358bdc5..01a899e6 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts @@ -186,10 +186,10 @@ export async function generateCompletions({ markdown, previousWarning, isExtractEndpoint, - model = getModel("gpt-4o-mini"), + model = getModel("gpt-4o-mini", "openai"), mode = "object", providerOptions, - retryModel = getModel("claude-3-7-sonnet-latest", "anthropic"), + retryModel = getModel("claude-3-5-sonnet-20240620", "anthropic"), }: GenerateCompletionsOptions): Promise<{ extract: any; numTokens: number;