+
+
+
## How to use it?
@@ -41,18 +77,26 @@ To use the API, you need to sign up on [Firecrawl](https://firecrawl.dev) and ge
Used to crawl a URL and all accessible subpages. This submits a crawl job and returns a job ID to check the status of the crawl.
```bash
-curl -X POST https://api.firecrawl.dev/v0/crawl \
+curl -X POST https://api.firecrawl.dev/v1/crawl \
-H 'Content-Type: application/json' \
- -H 'Authorization: Bearer YOUR_API_KEY' \
+ -H 'Authorization: Bearer fc-YOUR_API_KEY' \
-d '{
- "url": "https://mendable.ai"
+ "url": "https://docs.firecrawl.dev",
+ "limit": 100,
+ "scrapeOptions": {
+ "formats": ["markdown", "html"]
+ }
}'
```
-Returns a jobId
+Returns a crawl job id and the url to check the status of the crawl.
```json
-{ "jobId": "1234-5678-9101" }
+{
+ "success": true,
+ "id": "123-456-789",
+ "url": "https://api.firecrawl.dev/v1/crawl/123-456-789"
+}
```
### Check Crawl Job
@@ -60,7 +104,7 @@ Returns a jobId
Used to check the status of a crawl job and get its result.
```bash
-curl -X GET https://api.firecrawl.dev/v0/crawl/status/1234-5678-9101 \
+curl -X GET https://api.firecrawl.dev/v1/crawl/123-456-789 \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer YOUR_API_KEY'
```
@@ -68,18 +112,20 @@ curl -X GET https://api.firecrawl.dev/v0/crawl/status/1234-5678-9101 \
```json
{
"status": "completed",
- "current": 22,
- "total": 22,
+ "total": 36,
+ "creditsUsed": 36,
+ "expiresAt": "2024-00-00T00:00:00.000Z",
"data": [
{
- "content": "Raw Content ",
- "markdown": "# Markdown Content",
- "provider": "web-scraper",
+ "markdown": "[Firecrawl Docs home page!...",
+ "html": "...",
"metadata": {
- "title": "Mendable | AI for CX and Sales",
- "description": "AI for CX and Sales",
- "language": null,
- "sourceURL": "https://www.mendable.ai/"
+ "title": "Build a 'Chat with website' using Groq Llama 3 | Firecrawl",
+ "language": "en",
+ "sourceURL": "https://docs.firecrawl.dev/learn/rag-llama3",
+ "description": "Learn how to use Firecrawl, Groq Llama 3, and Langchain to build a 'Chat with your website' bot.",
+ "ogLocaleAlternate": [],
+ "statusCode": 200
}
}
]
@@ -88,14 +134,15 @@ curl -X GET https://api.firecrawl.dev/v0/crawl/status/1234-5678-9101 \
### Scraping
-Used to scrape a URL and get its content.
+Used to scrape a URL and get its content in the specified formats.
```bash
-curl -X POST https://api.firecrawl.dev/v0/scrape \
+curl -X POST https://api.firecrawl.dev/v1/scrape \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer YOUR_API_KEY' \
-d '{
- "url": "https://mendable.ai"
+ "url": "https://docs.firecrawl.dev",
+ "formats" : ["markdown", "html"]
}'
```
@@ -105,68 +152,95 @@ Response:
{
"success": true,
"data": {
- "content": "Raw Content ",
- "markdown": "# Markdown Content",
- "provider": "web-scraper",
+ "markdown": "Launch Week I is here! [See our Day 2 Release 🚀](https://www.firecrawl.dev/blog/launch-week-i-day-2-doubled-rate-limits)[💥 Get 2 months free...",
+ "html": " ": {
+ "type": "string"
+ },
+ "pageStatusCode": {
+ "type": "integer",
+ "description": "The status code of the page"
+ },
+ "pageError": {
+ "type": "string",
+ "nullable": true,
+ "description": "The error message of the page"
+ }
+
+ }
+ },
+ "llm_extraction": {
+ "type": "object",
+ "description": "Displayed when using LLM Extraction. Extracted data from the page following the schema defined.",
+ "nullable": true
+ },
+ "warning": {
+ "type": "string",
+ "nullable": true,
+ "description": "Can be displayed when using LLM Extraction. Warning message will let you know any issues with the extraction."
+ }
+ }
+ }
+ }
+ },
+ "CrawlStatusResponseObj": {
+ "type": "object",
+ "properties": {
+ "markdown": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "html": {
+ "type": "string",
+ "nullable": true,
+ "description": "HTML version of the content on page if `includeHtml` is true"
+ },
+ "rawHtml": {
+ "type": "string",
+ "nullable": true,
+ "description": "Raw HTML content of the page if `includeRawHtml` is true"
+ },
+ "index": {
+ "type": "integer",
+ "description": "The number of the page that was crawled. This is useful for `partial_data` so you know which page the data is from."
+ },
+ "metadata": {
+ "type": "object",
+ "properties": {
+ "title": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "language": {
+ "type": "string",
+ "nullable": true
+ },
+ "sourceURL": {
+ "type": "string",
+ "format": "uri"
+ },
+ "Hello, world!
'; + const expectedMarkdown = 'Hello, world!'; + await expect(parseMarkdown(html)).resolves.toBe(expectedMarkdown); + }); + + it('should convert complex HTML with nested elements to Markdown', async () => { + const html = 'Hello bold world!
Unclosed tag', expected: 'Unclosed tag' }, + { html: '
Wrong nesting
', expected: '**Wrong nesting**' }, + { html: 'Link without closing tag', expected: '[Link without closing tag](http://example.com)' } + ]; + + for (const { html, expected } of invalidHtmls) { + await expect(parseMarkdown(html)).resolves.toBe(expected); + } + }); +}); diff --git a/apps/api/src/lib/__tests__/job-priority.test.ts b/apps/api/src/lib/__tests__/job-priority.test.ts new file mode 100644 index 00000000..82477379 --- /dev/null +++ b/apps/api/src/lib/__tests__/job-priority.test.ts @@ -0,0 +1,134 @@ +import { + getJobPriority, + addJobPriority, + deleteJobPriority, +} from "../job-priority"; +import { redisConnection } from "../../services/queue-service"; +import { PlanType } from "../../types"; + +jest.mock("../../services/queue-service", () => ({ + redisConnection: { + sadd: jest.fn(), + srem: jest.fn(), + scard: jest.fn(), + expire: jest.fn(), + }, +})); + +describe("Job Priority Tests", () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + test("addJobPriority should add job_id to the set and set expiration", async () => { + const team_id = "team1"; + const job_id = "job1"; + await addJobPriority(team_id, job_id); + expect(redisConnection.sadd).toHaveBeenCalledWith( + `limit_team_id:${team_id}`, + job_id + ); + expect(redisConnection.expire).toHaveBeenCalledWith( + `limit_team_id:${team_id}`, + 60 + ); + }); + + test("deleteJobPriority should remove job_id from the set", async () => { + const team_id = "team1"; + const job_id = "job1"; + await deleteJobPriority(team_id, job_id); + expect(redisConnection.srem).toHaveBeenCalledWith( + `limit_team_id:${team_id}`, + job_id + ); + }); + + test("getJobPriority should return correct priority based on plan and set length", async () => { + const team_id = "team1"; + const plan: PlanType = "standard"; + (redisConnection.scard as jest.Mock).mockResolvedValue(150); + + const priority = await getJobPriority({ plan, team_id }); + expect(priority).toBe(10); + + (redisConnection.scard as jest.Mock).mockResolvedValue(250); + const priorityExceeded = await getJobPriority({ plan, team_id }); + expect(priorityExceeded).toBe(20); // basePriority + Math.ceil((250 - 200) * 0.4) + }); + + test("getJobPriority should handle different plans correctly", async () => { + const team_id = "team1"; + + (redisConnection.scard as jest.Mock).mockResolvedValue(50); + let plan: PlanType = "hobby"; + let priority = await getJobPriority({ plan, team_id }); + expect(priority).toBe(10); + + (redisConnection.scard as jest.Mock).mockResolvedValue(150); + plan = "hobby"; + priority = await getJobPriority({ plan, team_id }); + expect(priority).toBe(25); // basePriority + Math.ceil((150 - 50) * 0.3) + + (redisConnection.scard as jest.Mock).mockResolvedValue(25); + plan = "free"; + priority = await getJobPriority({ plan, team_id }); + expect(priority).toBe(10); + + (redisConnection.scard as jest.Mock).mockResolvedValue(60); + plan = "free"; + priority = await getJobPriority({ plan, team_id }); + expect(priority).toBe(28); // basePriority + Math.ceil((60 - 25) * 0.5) + }); + + test("addJobPriority should reset expiration time when adding new job", async () => { + const team_id = "team1"; + const job_id1 = "job1"; + const job_id2 = "job2"; + + await addJobPriority(team_id, job_id1); + expect(redisConnection.expire).toHaveBeenCalledWith( + `limit_team_id:${team_id}`, + 60 + ); + + // Clear the mock calls + (redisConnection.expire as jest.Mock).mockClear(); + + // Add another job + await addJobPriority(team_id, job_id2); + expect(redisConnection.expire).toHaveBeenCalledWith( + `limit_team_id:${team_id}`, + 60 + ); + }); + + test("Set should expire after 60 seconds", async () => { + const team_id = "team1"; + const job_id = "job1"; + + jest.useFakeTimers(); + + await addJobPriority(team_id, job_id); + expect(redisConnection.expire).toHaveBeenCalledWith( + `limit_team_id:${team_id}`, + 60 + ); + + // Fast-forward time by 59 seconds + jest.advanceTimersByTime(59000); + + // The set should still exist + expect(redisConnection.scard).not.toHaveBeenCalled(); + + // Fast-forward time by 2 more seconds (total 61 seconds) + jest.advanceTimersByTime(2000); + + // Check if the set has been removed (scard should return 0) + (redisConnection.scard as jest.Mock).mockResolvedValue(0); + const setSize = await redisConnection.scard(`limit_team_id:${team_id}`); + expect(setSize).toBe(0); + + jest.useRealTimers(); + }); +}); diff --git a/apps/api/src/lib/checkCredits.ts b/apps/api/src/lib/checkCredits.ts new file mode 100644 index 00000000..7e9d988d --- /dev/null +++ b/apps/api/src/lib/checkCredits.ts @@ -0,0 +1,32 @@ +import { checkTeamCredits } from "../services/billing/credit_billing"; +import { Logger } from "./logger"; + +type checkCreditsResponse = { + status: number; + error: string | null; +} + +export const checkCredits = async (team_id: string): Promise+ You can do whatever you want with the scrape results. Here is a + basic showcase of the markdown. +
+
+
+
+
+
+ 
+ 
+
+ {result.data.markdown.trim()}
+
+
+ ) : (
+ <>
+ + Failed to scrape this URL +
++ {result.toString()} +
+ + )} +