AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-07-10 21:01:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(batchExtract): thingymajig

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-15 20:28:53 -07:00
parent d119552eee
commit eea1267b50
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/lib/extract/completions/batchExtract.ts
View File

@ -1,4 +1,3 @@
import { logger } from "../../../lib/logger";
import { import {
generateCompletions, generateCompletions,
GenerateCompletionsOptions, GenerateCompletionsOptions,
@ -14,6 +13,7 @@ import { getModel } from "../../generic-ai";
import fs from "fs/promises"; import fs from "fs/promises";
import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape"; import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";
import type { Logger } from "winston";
type BatchExtractOptions = { type BatchExtractOptions = {
multiEntitySchema: any; multiEntitySchema: any;
@ -33,7 +33,7 @@ type BatchExtractOptions = {
* @param doc - The document to extract information from * @param doc - The document to extract information from
* @returns The completion promise * @returns The completion promise
*/ */
export async function batchExtractPromise(options: BatchExtractOptions): Promise<{ export async function batchExtractPromise(options: BatchExtractOptions, logger: Logger): Promise<{
extract: any; // array of extracted data extract: any; // array of extracted data
numTokens: number; numTokens: number;
totalUsage: TokenUsage; totalUsage: TokenUsage;
@ -82,7 +82,7 @@ export async function batchExtractPromise(options: BatchExtractOptions): Promise
smCallCount = smartScrapeCallCount; smCallCount = smartScrapeCallCount;
oCallCount = otherCallCount; oCallCount = otherCallCount;
} catch (error) { } catch (error) {
console.error(">>>>>>>error>>>>>\n", error); logger.error("extractData failed", { error });
} }
// await fs.writeFile( // await fs.writeFile(