From 199cbe8bcb3fc02584c44bfd3b33a656216f1713 Mon Sep 17 00:00:00 2001
From: Jeff Pereira <jeffluipe@gmail.com>
Date: Tue, 25 Jun 2024 12:20:25 -0700
Subject: [PATCH] add some types

---
 apps/api/src/main/runWebScraper.ts | 18 ++----------------
 apps/api/src/types.ts              | 24 ++++++++++++++++++++++--
 2 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index dee89bc4..e97e1413 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -1,5 +1,5 @@
 import { Job } from "bull";
-import { CrawlResult, WebScraperOptions } from "../types";
+import { CrawlResult, WebScraperOptions, RunWebScraperParams, RunWebScraperResult } from "../types";
 import { WebScraperDataProvider } from "../scraper/WebScraper";
 import { DocumentUrl, Progress } from "../lib/entities";
 import { billTeam } from "../services/billing/credit_billing";
@@ -45,21 +45,7 @@ export async function runWebScraper({
   onError,
   team_id,
   bull_job_id,
-}: {
-  url: string;
-  mode: "crawl" | "single_urls" | "sitemap";
-  crawlerOptions: any;
-  pageOptions?: any;
-  inProgress: (progress: any) => void;
-  onSuccess: (result: any) => void;
-  onError: (error: any) => void;
-  team_id: string;
-  bull_job_id: string;
-}): Promise<{
-  success: boolean;
-  message: string;
-  docs: Document[] | DocumentUrl[];
-}> {
+}: RunWebScraperParams): Promise<RunWebScraperResult> {
   try {
     const provider = new WebScraperDataProvider();
     if (mode === "crawl") {
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 971cc18a..e69353b6 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -1,4 +1,6 @@
-import { ExtractorOptions, Document } from "./lib/entities";
+import { ExtractorOptions, Document, DocumentUrl } from "./lib/entities";
+
+type Mode = "crawl" | "single_urls" | "sitemap";
 
 export interface CrawlResult {
   source: string;
@@ -20,13 +22,31 @@ export interface IngestResult {
 
 export interface WebScraperOptions {
   url: string;
-  mode: "crawl" | "single_urls" | "sitemap";
+  mode: Mode;
   crawlerOptions: any;
   pageOptions: any;
   team_id: string;
   origin?: string;
 }
 
+export interface RunWebScraperParams {
+  url: string;
+  mode: Mode;
+  crawlerOptions: any;
+  pageOptions?: any;
+  inProgress: (progress: any) => void;
+  onSuccess: (result: any) => void;
+  onError: (error: Error) => void;
+  team_id: string;
+  bull_job_id: string;
+}
+
+export interface RunWebScraperResult {
+  success: boolean;
+  message: string;
+  docs: Document[] | DocumentUrl[];
+}
+
 export interface FirecrawlJob {
   success: boolean;
   message: string;