From 388ce3cbce329de44c30189ce05616c892d38e18 Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Wed, 26 Jun 2024 21:15:42 -0300
Subject: [PATCH] Nick: small changes

---
 .../__tests__/removeUnwantedElements.test.ts  |  6 +--
 .../utils/removeUnwantedElements.ts           | 45 ++++++++++---------
 2 files changed, 28 insertions(+), 23 deletions(-)
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
index 23b9cb69..d17ca23c 100644
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
+++ b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
@@ -129,11 +129,11 @@ describe('removeUnwantedElements', () => {
     expect(result).not.toContain('<aside>');
   });
 
-  it('should handle no specified tags', () => {
-    const html = `<div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
+  it('should not handle no specified tags, return full content', () => {
+    const html = `<html><body><div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div></body></html>`;
     const options: PageOptions = { onlyIncludeTags: [] };
     const result = removeUnwantedElements(html, options);
-    expect(result).toBe('');
+    expect(result).toBe(html);
   });
 
   it('should handle specified tags as a string', () => {
diff --git a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
index 8b659889..38e4c5a0 100644
--- a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
@@ -2,46 +2,51 @@ import cheerio, { AnyNode, Cheerio } from "cheerio";
 import { PageOptions } from "../../../lib/entities";
 import { excludeNonMainTags } from "./excludeTags";
 
-export const removeUnwantedElements = (html: string, pageOptions: PageOptions) => {
+export const removeUnwantedElements = (
+  html: string,
+  pageOptions: PageOptions
+) => {
   const soup = cheerio.load(html);
 
   if (pageOptions.onlyIncludeTags) {
-    if (typeof pageOptions.onlyIncludeTags === 'string') {
+    if (typeof pageOptions.onlyIncludeTags === "string") {
       pageOptions.onlyIncludeTags = [pageOptions.onlyIncludeTags];
     }
-    // Create a new root element to hold the tags to keep
-    const newRoot = cheerio.load('<div></div>')('div');
-    pageOptions.onlyIncludeTags.forEach(tag => {
-      soup(tag).each((index, element) => {
-        newRoot.append(soup(element).clone());
+    if (pageOptions.onlyIncludeTags.length !== 0) {
+      // Create a new root element to hold the tags to keep
+      const newRoot = cheerio.load("<div></div>")("div");
+      pageOptions.onlyIncludeTags.forEach((tag) => {
+        soup(tag).each((index, element) => {
+          newRoot.append(soup(element).clone());
+        });
       });
-    });
-    return newRoot.html();
+      return newRoot.html();
+    }
   }
 
   soup("script, style, iframe, noscript, meta, head").remove();
-  
+
   if (pageOptions.removeTags) {
-    if (typeof pageOptions.removeTags === 'string') {
+    if (typeof pageOptions.removeTags === "string") {
       pageOptions.removeTags = [pageOptions.removeTags];
     }
-  
+
     if (Array.isArray(pageOptions.removeTags)) {
       pageOptions.removeTags.forEach((tag) => {
         let elementsToRemove: Cheerio<AnyNode>;
         if (tag.startsWith("*") && tag.endsWith("*")) {
           let classMatch = false;
 
-          const regexPattern = new RegExp(tag.slice(1, -1), 'i');
-          elementsToRemove = soup('*').filter((i, element) => {
-            if (element.type === 'tag') {
+          const regexPattern = new RegExp(tag.slice(1, -1), "i");
+          elementsToRemove = soup("*").filter((i, element) => {
+            if (element.type === "tag") {
               const attributes = element.attribs;
               const tagNameMatches = regexPattern.test(element.name);
-              const attributesMatch = Object.keys(attributes).some(attr => 
+              const attributesMatch = Object.keys(attributes).some((attr) =>
                 regexPattern.test(`${attr}="${attributes[attr]}"`)
               );
-              if (tag.startsWith('*.')) {
-                classMatch = Object.keys(attributes).some(attr => 
+              if (tag.startsWith("*.")) {
+                classMatch = Object.keys(attributes).some((attr) =>
                   regexPattern.test(`class="${attributes[attr]}"`)
                 );
               }
@@ -56,7 +61,7 @@ export const removeUnwantedElements = (html: string, pageOptions: PageOptions) =
       });
     }
   }
-  
+
   if (pageOptions.onlyMainContent) {
     excludeNonMainTags.forEach((tag) => {
       const elementsToRemove = soup(tag);
@@ -65,4 +70,4 @@ export const removeUnwantedElements = (html: string, pageOptions: PageOptions) =
   }
   const cleanedHtml = soup.html();
   return cleanedHtml;
-};
\ No newline at end of file
+};
