AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 01:19:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

add some types

Browse Source
This commit is contained in:
Jeff Pereira 2024-06-25 12:20:25 -07:00
parent f84fb4b331
commit 199cbe8bcb
2 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/main/runWebScraper.ts
View File

@ -1,5 +1,5 @@
import { Job } from "bull"; import { Job } from "bull";
import { CrawlResult, WebScraperOptions } from "../types"; import { CrawlResult, WebScraperOptions, RunWebScraperParams, RunWebScraperResult } from "../types";
import { WebScraperDataProvider } from "../scraper/WebScraper"; import { WebScraperDataProvider } from "../scraper/WebScraper";
import { DocumentUrl, Progress } from "../lib/entities"; import { DocumentUrl, Progress } from "../lib/entities";
import { billTeam } from "../services/billing/credit_billing"; import { billTeam } from "../services/billing/credit_billing";
@ -45,21 +45,7 @@ export async function runWebScraper({
onError, onError,
team_id, team_id,
bull_job_id, bull_job_id,
}: { }: RunWebScraperParams): Promise<RunWebScraperResult> {
url: string;
mode: "crawl" | "single_urls" | "sitemap";
crawlerOptions: any;
pageOptions?: any;
inProgress: (progress: any) => void;
onSuccess: (result: any) => void;
onError: (error: any) => void;
team_id: string;
bull_job_id: string;
}): Promise<{
success: boolean;
message: string;
docs: Document[] | DocumentUrl[];
}> {
try { try {
const provider = new WebScraperDataProvider(); const provider = new WebScraperDataProvider();
if (mode === "crawl") { if (mode === "crawl") {

24
apps/api/src/types.ts
View File

@ -1,4 +1,6 @@
import { ExtractorOptions, Document } from "./lib/entities"; import { ExtractorOptions, Document, DocumentUrl } from "./lib/entities";
type Mode = "crawl" | "single_urls" | "sitemap";
export interface CrawlResult { export interface CrawlResult {
source: string; source: string;
@ -20,13 +22,31 @@ export interface IngestResult {
export interface WebScraperOptions { export interface WebScraperOptions {
url: string; url: string;
mode: "crawl" | "single_urls" | "sitemap"; mode: Mode;
crawlerOptions: any; crawlerOptions: any;
pageOptions: any; pageOptions: any;
team_id: string; team_id: string;
origin?: string; origin?: string;
} }
export interface RunWebScraperParams {
url: string;
mode: Mode;
crawlerOptions: any;
pageOptions?: any;
inProgress: (progress: any) => void;
onSuccess: (result: any) => void;
onError: (error: Error) => void;
team_id: string;
bull_job_id: string;
}
export interface RunWebScraperResult {
success: boolean;
message: string;
docs: Document[] | DocumentUrl[];
}
export interface FirecrawlJob { export interface FirecrawlJob {
success: boolean; success: boolean;
message: string; message: string;