diff --git a/apps/api/.gitignore b/apps/api/.gitignore index 66bccfed..edc2faf4 100644 --- a/apps/api/.gitignore +++ b/apps/api/.gitignore @@ -3,4 +3,6 @@ .env *.csv dump.rdb -/mongo-data \ No newline at end of file +/mongo-data + +/.next/ diff --git a/apps/ui/ingestion-ui/src/App.css b/apps/ui/ingestion-ui/src/App.css deleted file mode 100644 index b9d355df..00000000 --- a/apps/ui/ingestion-ui/src/App.css +++ /dev/null @@ -1,42 +0,0 @@ -#root { - max-width: 1280px; - margin: 0 auto; - padding: 2rem; - text-align: center; -} - -.logo { - height: 6em; - padding: 1.5em; - will-change: filter; - transition: filter 300ms; -} -.logo:hover { - filter: drop-shadow(0 0 2em #646cffaa); -} -.logo.react:hover { - filter: drop-shadow(0 0 2em #61dafbaa); -} - -@keyframes logo-spin { - from { - transform: rotate(0deg); - } - to { - transform: rotate(360deg); - } -} - -@media (prefers-reduced-motion: no-preference) { - a:nth-of-type(2) .logo { - animation: logo-spin infinite 20s linear; - } -} - -.card { - padding: 2em; -} - -.read-the-docs { - color: #888; -} diff --git a/apps/ui/ingestion-ui/src/App.tsx b/apps/ui/ingestion-ui/src/App.tsx index f0d98a3a..eb0e6954 100644 --- a/apps/ui/ingestion-ui/src/App.tsx +++ b/apps/ui/ingestion-ui/src/App.tsx @@ -1,4 +1,3 @@ -import "./App.css"; import FirecrawlComponent from "./components/ingestion"; function App() { diff --git a/apps/ui/ingestion-ui/src/assets/react.svg b/apps/ui/ingestion-ui/src/assets/react.svg deleted file mode 100644 index 6c87de9b..00000000 --- a/apps/ui/ingestion-ui/src/assets/react.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/apps/ui/ingestion-ui/src/components/ingestion.tsx b/apps/ui/ingestion-ui/src/components/ingestion.tsx index a17ee43a..a7cd9722 100644 --- a/apps/ui/ingestion-ui/src/components/ingestion.tsx +++ b/apps/ui/ingestion-ui/src/components/ingestion.tsx @@ -15,11 +15,12 @@ import { CollapsibleContent, CollapsibleTrigger, } from "@/components/ui/collapsible"; -import { ChevronDown } from "lucide-react"; +import { ChevronDown, ChevronLeft, ChevronRight } from "lucide-react"; -// Hardcoded values (not recommended for production) +//! Hardcoded values (not recommended for production) +//! Highly recommended to move all Firecrawl API calls to the backend (e.g. Next.js API route) const FIRECRAWL_API_URL = "https://api.firecrawl.dev"; // Replace with your actual API URL whether it is local or using Firecrawl Cloud -const FIRECRAWL_API_KEY = ""; // Replace with your actual API key +const FIRECRAWL_API_KEY = "fc-YOUR_API_KEY"; // Replace with your actual API key interface FormData { url: string; @@ -100,7 +101,8 @@ export default function FirecrawlComponent() { const [elapsedTime, setElapsedTime] = useState(0); const [showCrawlStatus, setShowCrawlStatus] = useState(false); const [isScraping, setIsScraping] = useState(false); - const [showAllUrls, setShowAllUrls] = useState(false); + const [currentPage, setCurrentPage] = useState(1); + const urlsPerPage = 10; useEffect(() => { let timer: NodeJS.Timeout; @@ -289,6 +291,7 @@ export default function FirecrawlComponent() { const data: ScrapeResult = await response.json(); newScrapeResults[url] = data; setCrawlStatus((prev) => ({ ...prev, current: index + 1 })); + setScrapeResults({ ...scrapeResults, ...newScrapeResults }); } catch (error) { console.error(`Error scraping ${url}:`, error); newScrapeResults[url] = { @@ -312,22 +315,35 @@ export default function FirecrawlComponent() { } } - setScrapeResults(newScrapeResults); setLoading(false); setIsScraping(false); }; + const handlePageChange = (newPage: number) => { + setCurrentPage(newPage); + }; + + const paginatedUrls = crawledUrls.slice( + (currentPage - 1) * urlsPerPage, + currentPage * urlsPerPage + ); + return (
- - - Extract web content with Firecrawl 🔥 + + + Extract web content + + Powered by Firecrawl 🔥 +
- Use this component to quickly build your own UI for Firecrawl. Plug - in your API key and the component will handle the rest. Learn more - on the{" "} + Use this component to quickly give your users the ability to connect + their AI apps to web data with Firecrawl. Learn more on the{" "}
- +
{selectedUrls.length === crawledUrls.length - ? "Unselect All" - : "Select All"} + ? `Unselect All (${selectedUrls.length})` + : `Select All (${selectedUrls.length})`} )} @@ -503,41 +548,57 @@ export default function FirecrawlComponent() { !isScraping && ( <>
    - {(showAllUrls ? crawledUrls : crawledUrls.slice(0, 10)).map( - (url, index) => ( -
  • - - setSelectedUrls((prev) => - prev.includes(url) - ? prev.filter((u) => u !== url) - : [...prev, url] - ) - } - /> - {url} -
    • - ) - )} -
- {crawledUrls.length > 10 && ( -
- -
- )} + + setSelectedUrls((prev) => + prev.includes(url) + ? prev.filter((u) => u !== url) + : [...prev, url] + ) + } + /> + + {url.length > 70 ? `${url.slice(0, 70)}...` : url} + + + ))} + +
+ + + Page {currentPage} of{" "} + {Math.ceil(crawledUrls.length / urlsPerPage)} + + +
)} - + {crawledUrls.length > 0 && !scrapingSelectedLoading && ( - ))}