Merge pull request #867 from mendableai/fix/remove-base64-images

[Feat] Added remove base64 images options (true by default)

apps/api/src/controllers/v1/types.ts

@@ -141,6 +141,7 @@ export const scrapeOptions = z.object({
     languages: z.string().array().optional(),
   }).optional(),
   skipTlsVerification: z.boolean().default(false),
+  removeBase64Images: z.boolean().default(true),
 }).strict(strictMessage)
 
 
@@ -470,6 +471,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions {
     actions: x.actions as Action[], // no strict null checking grrrr - mogery
     geolocation: x.location ?? x.geolocation,
     skipTlsVerification: x.skipTlsVerification,
+    removeBase64Images: x.removeBase64Images,
     mobile: x.mobile,
   };
 }

apps/api/src/lib/entities.ts

@@ -58,6 +58,7 @@ export type PageOptions = {
     country?: string;
   };
   skipTlsVerification?: boolean;
+  removeBase64Images?: boolean;
   mobile?: boolean;
 };
 

apps/api/src/scraper/WebScraper/index.ts

@@ -595,6 +595,7 @@ export class WebScraperDataProvider {
       actions: options.pageOptions?.actions ?? undefined,
       geolocation: options.pageOptions?.geolocation ?? undefined,
       skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,
+      removeBase64Images: options.pageOptions?.removeBase64Images ?? true,
       mobile: options.pageOptions?.mobile ?? false,
     };
     this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };

apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts

@@ -28,7 +28,7 @@ export async function scrapWithFireEngine({
   waitFor = 0,
   screenshot = false,
   fullPageScreenshot = false,
-  pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, mobile: false },
+  pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true, mobile: false },
   fireEngineOptions = {},
   headers,
   options,
@@ -40,7 +40,7 @@ export async function scrapWithFireEngine({
   waitFor?: number;
   screenshot?: boolean;
   fullPageScreenshot?: boolean;
-  pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, mobile?: boolean };
+  pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean, mobile?: boolean };
   fireEngineOptions?: FireEngineOptions;
   headers?: Record<string, string>;
   options?: any;
@@ -121,6 +121,7 @@ export async function scrapWithFireEngine({
           scrollXPaths: pageOptions?.scrollXPaths ?? [],
           geolocation: pageOptions?.geolocation,
           skipTlsVerification: pageOptions?.skipTlsVerification ?? false,
+          removeBase64Images: pageOptions?.removeBase64Images ?? true,
           actions: actions,
         },
         {

apps/api/src/scraper/WebScraper/single_url.ts

@@ -22,6 +22,7 @@ import { Logger } from "../../lib/logger";
 import { ScrapeEvents } from "../../lib/scrape-events";
 import { clientSideError } from "../../strings";
 import { ScrapeActionContent } from "../../lib/entities";
+import { removeBase64Images } from "./utils/removeBase64Images";
 
 dotenv.config();
 
@@ -159,6 +160,7 @@ export async function scrapSingleUrl(
     actions: pageOptions.actions ?? undefined,
     geolocation: pageOptions.geolocation ?? undefined,
     skipTlsVerification: pageOptions.skipTlsVerification ?? false,
+    removeBase64Images: pageOptions.removeBase64Images ?? true,
     mobile: pageOptions.mobile ?? false,
   }
 
@@ -351,7 +353,10 @@ export async function scrapSingleUrl(
     }
     //* TODO: add an optional to return markdown or structured/extracted content
     let cleanedHtml = removeUnwantedElements(scraperResponse.text, pageOptions);
-    const text = await parseMarkdown(cleanedHtml);
+    let text = await parseMarkdown(cleanedHtml);
+    if (pageOptions.removeBase64Images) {
+      text = await removeBase64Images(text);
+    }
 
     const insertedLogId = await logInsertPromise;
     ScrapeEvents.updateScrapeResult(insertedLogId, {

apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts

@@ -0,0 +1,7 @@
+export const removeBase64Images = async (
+  markdown: string,
+) => {
+  const regex = /(!\[.*?\])\(data:image\/.*?;base64,.*?\)/g;
+  markdown = markdown.replace(regex, '$1(<Base64-Image-Removed>)');
+  return markdown;
+};