Nick: changes to the ui component

apps/api/.gitignore

@@ -3,4 +3,6 @@
 .env
 *.csv
 dump.rdb
-/mongo-data
+/mongo-data
+
+/.next/

apps/ui/ingestion-ui/src/App.css

@@ -1,42 +0,0 @@
-#root {
-  max-width: 1280px;
-  margin: 0 auto;
-  padding: 2rem;
-  text-align: center;
-}
-
-.logo {
-  height: 6em;
-  padding: 1.5em;
-  will-change: filter;
-  transition: filter 300ms;
-}
-.logo:hover {
-  filter: drop-shadow(0 0 2em #646cffaa);
-}
-.logo.react:hover {
-  filter: drop-shadow(0 0 2em #61dafbaa);
-}
-
-@keyframes logo-spin {
-  from {
-    transform: rotate(0deg);
-  }
-  to {
-    transform: rotate(360deg);
-  }
-}
-
-@media (prefers-reduced-motion: no-preference) {
-  a:nth-of-type(2) .logo {
-    animation: logo-spin infinite 20s linear;
-  }
-}
-
-.card {
-  padding: 2em;
-}
-
-.read-the-docs {
-  color: #888;
-}

apps/ui/ingestion-ui/src/App.tsx

@@ -1,4 +1,3 @@
-import "./App.css";
 import FirecrawlComponent from "./components/ingestion";
 
 function App() {

apps/ui/ingestion-ui/src/assets/react.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

apps/ui/ingestion-ui/src/components/ingestion.tsx

@@ -15,11 +15,12 @@ import {
   CollapsibleContent,
   CollapsibleTrigger,
 } from "@/components/ui/collapsible";
-import { ChevronDown } from "lucide-react";
+import { ChevronDown, ChevronLeft, ChevronRight } from "lucide-react";
 
-// Hardcoded values (not recommended for production)
+//! Hardcoded values (not recommended for production)
+//! Highly recommended to move all Firecrawl API calls to the backend (e.g. Next.js API route)
 const FIRECRAWL_API_URL = "https://api.firecrawl.dev"; // Replace with your actual API URL whether it is local or using Firecrawl Cloud
-const FIRECRAWL_API_KEY = ""; // Replace with your actual API key
+const FIRECRAWL_API_KEY = "fc-YOUR_API_KEY"; // Replace with your actual API key
 
 interface FormData {
   url: string;
@@ -100,7 +101,8 @@ export default function FirecrawlComponent() {
   const [elapsedTime, setElapsedTime] = useState<number>(0);
   const [showCrawlStatus, setShowCrawlStatus] = useState<boolean>(false);
   const [isScraping, setIsScraping] = useState<boolean>(false);
-  const [showAllUrls, setShowAllUrls] = useState<boolean>(false);
+  const [currentPage, setCurrentPage] = useState<number>(1);
+  const urlsPerPage = 10;
 
   useEffect(() => {
     let timer: NodeJS.Timeout;
@@ -289,6 +291,7 @@ export default function FirecrawlComponent() {
         const data: ScrapeResult = await response.json();
         newScrapeResults[url] = data;
         setCrawlStatus((prev) => ({ ...prev, current: index + 1 }));
+        setScrapeResults({ ...scrapeResults, ...newScrapeResults });
       } catch (error) {
         console.error(`Error scraping ${url}:`, error);
         newScrapeResults[url] = {
@@ -312,22 +315,35 @@ export default function FirecrawlComponent() {
       }
     }
 
-    setScrapeResults(newScrapeResults);
     setLoading(false);
     setIsScraping(false);
   };
 
+  const handlePageChange = (newPage: number) => {
+    setCurrentPage(newPage);
+  };
+
+  const paginatedUrls = crawledUrls.slice(
+    (currentPage - 1) * urlsPerPage,
+    currentPage * urlsPerPage
+  );
+
   return (
     <div className="max-w-2xl mx-auto p-4">
       <Card>
-        <CardHeader className="flex items-center justify-between">
+        <CardHeader className="flex items-start justify-between mb-0 pb-4">
-          <CardTitle className="flex items-center space-x-2">
+          <CardTitle className="flex items-center justify-between w-full space-x-2">
-            <span>Extract web content with Firecrawl 🔥</span>
+            <span className="text-base">Extract web content</span>
+            <a
+              href="https://www.firecrawl.dev"
+              className="text-xs text-gray-500 font-normal px-3 py-1 bg-zinc-100 rounded-xl hover:bg-zinc-200 transition-colors"
+            >
+              Powered by Firecrawl 🔥
+            </a>
           </CardTitle>
           <div className="text-sm text-gray-500 w-11/12 items-center">
-            Use this component to quickly build your own UI for Firecrawl. Plug
+            Use this component to quickly give your users the ability to connect
-            in your API key and the component will handle the rest. Learn more
+            their AI apps to web data with Firecrawl. Learn more on the{" "}
-            on the{" "}
             <a
               href="https://docs.firecrawl.dev/"
               className="text-sm text-blue-500"
@@ -347,7 +363,36 @@ export default function FirecrawlComponent() {
                 onChange={handleChange}
               />
               <Button type="submit" variant="default" disabled={loading}>
-                {loading ? "Running..." : "Run"}
+                {loading ? (
+                  <div
+                    role="status"
+                    className="flex items-center justify-between space-x-2"
+                  >
+                    <svg
+                      className="animate-spin  h-4 w-4 text-white"
+                      xmlns="http://www.w3.org/2000/svg"
+                      fill="none"
+                      viewBox="0 0 24 24"
+                    >
+                      <circle
+                        className="opacity-25"
+                        cx="12"
+                        cy="12"
+                        r="10"
+                        stroke="currentColor"
+                        strokeWidth="4"
+                      ></circle>
+                      <path
+                        className="opacity-75"
+                        fill="currentColor"
+                        d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
+                      ></path>
+                    </svg>
+                    <span className="sr-only">Loading...</span>
+                  </div>
+                ) : (
+                  "Run"
+                )}
               </Button>
             </div>
             <Collapsible
@@ -361,7 +406,7 @@ export default function FirecrawlComponent() {
                   <ChevronDown className="h-4 w-4 opacity-50" />
                 </Button>
               </CollapsibleTrigger>
-              <CollapsibleContent className="space-y-4 mt-4">
+              <CollapsibleContent className="space-y-4 mt-4 px-2">
                 <div className="flex items-center space-x-2">
                   <Checkbox
                     id="crawlSubPages"
@@ -484,8 +529,8 @@ export default function FirecrawlComponent() {
                         className="text-sm cursor-pointer"
                       >
                         {selectedUrls.length === crawledUrls.length
-                          ? "Unselect All"
+                          ? `Unselect All (${selectedUrls.length})`
-                          : "Select All"}
+                          : `Select All (${selectedUrls.length})`}
                       </label>
                     </>
                   )}
@@ -503,41 +548,57 @@ export default function FirecrawlComponent() {
             !isScraping && (
               <>
                 <ul className="pl-2">
-                  {(showAllUrls ? crawledUrls : crawledUrls.slice(0, 10)).map(
+                  {paginatedUrls.map((url, index) => (
-                    (url, index) => (
+                    <li
-                      <li key={index} className="flex items-center space-x-2">
+                      key={index}
-                        <Checkbox
+                      className="flex items-center space-x-2 my-2 text-sm"
-                          checked={selectedUrls.includes(url)}
-                          onCheckedChange={() =>
-                            setSelectedUrls((prev) =>
-                              prev.includes(url)
-                                ? prev.filter((u) => u !== url)
-                                : [...prev, url]
-                            )
-                          }
-                        />
-                        <span>{url}</span>
-                      </li>
-                    )
-                  )}
-                </ul>
-                {crawledUrls.length > 10 && (
-                  <div className="flex justify-center mt-2">
-                    <Button
-                      variant="link"
-                      onClick={() => setShowAllUrls(!showAllUrls)}
                     >
-                      {showAllUrls ? "Show Less" : "Show All"}
+                      <Checkbox
-                    </Button>
+                        checked={selectedUrls.includes(url)}
-                  </div>
+                        onCheckedChange={() =>
-                )}
+                          setSelectedUrls((prev) =>
+                            prev.includes(url)
+                              ? prev.filter((u) => u !== url)
+                              : [...prev, url]
+                          )
+                        }
+                      />
+                      <span className="flex items-center max-w-lg">
+                        {url.length > 70 ? `${url.slice(0, 70)}...` : url}
+                      </span>
+                    </li>
+                  ))}
+                </ul>
+                <div className="flex items-center justify-between mt-4">
+                  <Button
+                    variant="outline"
+                    className="px-2"
+                    onClick={() => handlePageChange(currentPage - 1)}
+                    disabled={currentPage === 1}
+                  >
+                    <ChevronLeft className="h-5 w-5" />
+                  </Button>
+                  <span className="text-sm text-gray-500">
+                    Page {currentPage} of{" "}
+                    {Math.ceil(crawledUrls.length / urlsPerPage)}
+                  </span>
+                  <Button
+                    variant="outline"
+                    className="px-2"
+                    onClick={() => handlePageChange(currentPage + 1)}
+                    disabled={currentPage * urlsPerPage >= crawledUrls.length}
+                  >
+                    <ChevronRight className="h-5 w-5 " />
+                  </Button>
+                </div>
               </>
             )}
         </CardContent>
-        <CardFooter className="flex justify-center">
+        <CardFooter className="w-full flex justify-center">
           {crawledUrls.length > 0 && !scrapingSelectedLoading && (
             <Button
               variant="default"
+              className="w-full"
               onClick={handleScrapeSelected}
               disabled={loading || selectedUrls.length === 0}
             >
@@ -549,17 +610,24 @@ export default function FirecrawlComponent() {
 
       {Object.keys(scrapeResults).length > 0 && (
         <div className="mt-4">
-          <h2 className="text-2xl font-bold mb-4">Scrape Results</h2>
+          <h2 className="text-base font-bold ">Scrape Results</h2>
-          <div className="grid grid-cols-2 gap-4">
+          <p className="text-sm text-gray-500">
+            You can do whatever you want with the scrape results. Here is a
+            basic showcase of the markdown.
+          </p>
+          <div className="flex flex-col gap-4 mt-4 w-full">
             {Object.entries(scrapeResults).map(([url, result]) => (
-              <Card key={url} className="overflow-hidden">
+              <Card key={url} className="relative p-4 w-full">
-                <CardContent>
+                <CardTitle className="text-sm font-normal flex flex-col">
-                  <div className="max-h-60 overflow-y-auto">
+                  <span>{result.data.metadata.title}</span>
-                    <div className="text-base font-bold py-2">
+                  <span className="text-xs text-gray-500">
-                      {url
+                    {url
-                        .replace(/^(https?:\/\/)?(www\.)?/, "")
+                      .replace(/^(https?:\/\/)?(www\.)?/, "")
-                        .replace(/\/$/, "")}
+                      .replace(/\/$/, "")}
-                    </div>
+                  </span>
+                </CardTitle>
+                <CardContent className="relative px-0 pt-2 !text-xs w-full">
+                  <div className=" overflow-y-auto bg-zinc-100 rounded-md p-2 w-full">
                     {result.success ? (
                       <>
                         <pre className="text-xs whitespace-pre-wrap">
@@ -567,27 +635,17 @@ export default function FirecrawlComponent() {
                         </pre>
                       </>
                     ) : (
-                      <p className="text-red-500">Failed to scrape this URL</p>
+                      <>
+                        <p className="text-red-500">
+                          Failed to scrape this URL
+                        </p>
+                        <p className="text-zinc-500 font-mono">
+                          {result.toString()}
+                        </p>
+                      </>
                     )}
                   </div>
                 </CardContent>
-                <CardFooter className="flex justify-center">
-                  <Button
-                    variant="outline"
-                    size="sm"
-                    onClick={(event) => {
-                      navigator.clipboard.writeText(result.data.markdown);
-                      const button = event.currentTarget as HTMLButtonElement;
-                      const originalText = button.textContent;
-                      button.textContent = "Copied!";
-                      setTimeout(() => {
-                        button.textContent = originalText;
-                      }, 2000);
-                    }}
-                  >
-                    Copy Markdown
-                  </Button>
-                </CardFooter>
               </Card>
             ))}
           </div>