mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-07-10 21:01:51 +08:00
feat(batchExtract): thingymajig
This commit is contained in:
parent
d119552eee
commit
eea1267b50
@ -1,4 +1,3 @@
|
|||||||
import { logger } from "../../../lib/logger";
|
|
||||||
import {
|
import {
|
||||||
generateCompletions,
|
generateCompletions,
|
||||||
GenerateCompletionsOptions,
|
GenerateCompletionsOptions,
|
||||||
@ -14,6 +13,7 @@ import { getModel } from "../../generic-ai";
|
|||||||
|
|
||||||
import fs from "fs/promises";
|
import fs from "fs/promises";
|
||||||
import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";
|
import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";
|
||||||
|
import type { Logger } from "winston";
|
||||||
|
|
||||||
type BatchExtractOptions = {
|
type BatchExtractOptions = {
|
||||||
multiEntitySchema: any;
|
multiEntitySchema: any;
|
||||||
@ -33,7 +33,7 @@ type BatchExtractOptions = {
|
|||||||
* @param doc - The document to extract information from
|
* @param doc - The document to extract information from
|
||||||
* @returns The completion promise
|
* @returns The completion promise
|
||||||
*/
|
*/
|
||||||
export async function batchExtractPromise(options: BatchExtractOptions): Promise<{
|
export async function batchExtractPromise(options: BatchExtractOptions, logger: Logger): Promise<{
|
||||||
extract: any; // array of extracted data
|
extract: any; // array of extracted data
|
||||||
numTokens: number;
|
numTokens: number;
|
||||||
totalUsage: TokenUsage;
|
totalUsage: TokenUsage;
|
||||||
@ -82,7 +82,7 @@ export async function batchExtractPromise(options: BatchExtractOptions): Promise
|
|||||||
smCallCount = smartScrapeCallCount;
|
smCallCount = smartScrapeCallCount;
|
||||||
oCallCount = otherCallCount;
|
oCallCount = otherCallCount;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(">>>>>>>error>>>>>\n", error);
|
logger.error("extractData failed", { error });
|
||||||
}
|
}
|
||||||
|
|
||||||
// await fs.writeFile(
|
// await fs.writeFile(
|
||||||
|
Loading…
x
Reference in New Issue
Block a user