AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 23:25:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/save-docs-on-supabase

Browse Source
This commit is contained in:
Nicolas 2024-07-05 12:27:22 -03:00
commit 914897c9d2
40 changed files with 3547 additions and 2085 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,57 @@
version: 2
updates:
# playwright-service
- package-ecosystem: "pip"
directory: "/apps/playwright-service"
schedule:
interval: "weekly"
groups:
prod-deps:
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message:
prefix: "apps/playwright-service"
include: "scope"
# python-sdk
- package-ecosystem: "pip"
directory: "/apps/python-sdk"
schedule:
interval: "weekly"
groups:
prod-deps:
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message:
prefix: "apps/python-sdk"
include: "scope"
# api
- package-ecosystem: "npm"
directory: "/apps/api"
schedule:
interval: "weekly"
groups:
prod-deps:
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message:
prefix: "apps/api"
include: "scope"
# test-suite
- package-ecosystem: "npm"
directory: "/apps/test-suite"
schedule:
interval: "weekly"
groups:
prod-deps:
dependency-type: "production"
dev-deps:
dependency-type: "development"
commit-message:
prefix: "apps/test-suite"
include: "scope"

3
.gitignore vendored
View File

@ -15,3 +15,6 @@ apps/test-suite/node_modules/
apps/test-suite/.env
apps/test-suite/logs
apps/test-suite/load-test-results/test-run-report.json
apps/playwright-service-ts/node_modules/
apps/playwright-service-ts/package-lock.json

50
SELF_HOST.md
View File

@ -1,34 +1,64 @@
# Self-hosting Firecrawl
*We're currently working on a more in-depth guide on how to self-host, but in the meantime, here is a simplified version.*
## Self-hosting Firecrawl
_We're currently working on a more in-depth guide on how to self-host, but in the meantime, here is a simplified version._
Refer to [CONTRIBUTING.md](https://github.com/mendableai/firecrawl/blob/main/CONTRIBUTING.md) for instructions on how to run it locally.
## Getting Started
First, clone this repository and copy the example env file from api folder `.env.example` to `.env`.
First, clone this repository and copy the example env file from the API folder `.env.example` to `.env`.
### Steps
1. Clone the repository:
```bash
git clone https://github.com/mendableai/firecrawl.git
cd firecrawl
cp ./apps/api/.env.example ./.env
```
For running the simplest version of FireCrawl, edit the `USE_DB_AUTHENTICATION` on `.env` to not use the database authentication.
```yml
2. For running the simplest version of FireCrawl, edit the `USE_DB_AUTHENTICATION` in `.env` to not use the database authentication:
```plaintext
USE_DB_AUTHENTICATION=false
```
Update the Redis URL in the .env file to align with the Docker configuration:
```yml
3. Update the Redis URL in the .env file to align with the Docker configuration:
```plaintext
REDIS_URL=redis://redis:6379
```
Once that's complete, you can simply run the following commands to get started:
4. #### Option: Running with TypeScript Playwright Service
* Update the `docker-compose.yml` file to change the Playwright service:
```plaintext
build: apps/playwright-service
```
TO
```plaintext
build: apps/playwright-service-ts
```
* Set the `PLAYWRIGHT_MICROSERVICE_URL` in your `.env` file:
```plaintext
PLAYWRIGHT_MICROSERVICE_URL=http://localhost:3000/scrape
```
* Don't forget to set the proxy server in your `.env` file as needed.
5. Build and run the Docker containers:
```bash
docker compose build
docker compose up
```
This will run a local instance of Firecrawl which can be accessed at `http://localhost:3002`.
# Install Firecrawl on a Kubernetes Cluster (Simple Version)
Read the [examples/kubernetes-cluster-install/README.md](examples/kubernetes-cluster-install/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster.
## Install Firecrawl on a Kubernetes Cluster (Simple Version)
Read the [examples/kubernetes-cluster-install/README.md](https://github.com/mendableai/firecrawl/blob/main/examples/kubernetes-cluster-install/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster.

241
apps/api/openapi.json
View File

@ -68,9 +68,21 @@
},
"description": "Tags, classes and ids to remove from the page. Use comma separated values. Example: 'script, .ad, #footer'"
},
"onlyIncludeTags": {
"type": "array",
"items": {
"type": "string"
},
"description": "Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: 'script, .ad, #footer'"
},
"headers": {
"type": "object",
"description": "Headers to send with the request. Can be used to send cookies, user-agent, etc."
},
"replaceAllPathsWithAbsolutePaths": {
"type": "boolean",
"description": "Replace all relative paths with absolute paths for images and links",
"default": false
}
}
},
@ -80,8 +92,8 @@
"properties": {
"mode": {
"type": "string",
"enum": ["llm-extraction"],
"description": "The extraction mode to use, currently supports 'llm-extraction'"
"enum": ["llm-extraction", "llm-extraction-from-raw-html"],
"description": "The extraction mode to use. llm-extraction: Extracts information from the cleaned and parsed content. llm-extraction-from-raw-html: Extracts information directly from the raw HTML."
},
"extractionPrompt": {
"type": "string",
@ -184,7 +196,7 @@
},
"maxDepth": {
"type": "integer",
"description": "Maximum depth to crawl. Depth 1 is the base URL, depth 2 is the base URL and its direct children, and so on."
"description": "Maximum depth to crawl relative to the entered URL. A maxDepth of 0 scrapes only the entered URL. A maxDepth of 1 scrapes the entered URL and all pages one level deep. A maxDepth of 2 scrapes the entered URL and all pages up to two levels deep. Higher values follow the same pattern."
},
"mode": {
"type": "string",
@ -526,118 +538,13 @@
"type": "string",
"nullable": true
},
"keywords": {
"type": "string",
"nullable": true
},
"robots": {
"type": "string",
"nullable": true
},
"ogTitle": {
"type": "string",
"nullable": true
},
"ogDescription": {
"type": "string",
"nullable": true
},
"ogUrl": {
"type": "string",
"format": "uri",
"nullable": true
},
"ogImage": {
"type": "string",
"nullable": true
},
"ogAudio": {
"type": "string",
"nullable": true
},
"ogDeterminer": {
"type": "string",
"nullable": true
},
"ogLocale": {
"type": "string",
"nullable": true
},
"ogLocaleAlternate": {
"type": "array",
"items": {
"type": "string"
},
"nullable": true
},
"ogSiteName": {
"type": "string",
"nullable": true
},
"ogVideo": {
"type": "string",
"nullable": true
},
"dctermsCreated": {
"type": "string",
"nullable": true
},
"dcDateCreated": {
"type": "string",
"nullable": true
},
"dcDate": {
"type": "string",
"nullable": true
},
"dctermsType": {
"type": "string",
"nullable": true
},
"dcType": {
"type": "string",
"nullable": true
},
"dctermsAudience": {
"type": "string",
"nullable": true
},
"dctermsSubject": {
"type": "string",
"nullable": true
},
"dcSubject": {
"type": "string",
"nullable": true
},
"dcDescription": {
"type": "string",
"nullable": true
},
"dctermsKeywords": {
"type": "string",
"nullable": true
},
"modifiedTime": {
"type": "string",
"nullable": true
},
"publishedTime": {
"type": "string",
"nullable": true
},
"articleTag": {
"type": "string",
"nullable": true
},
"articleSection": {
"type": "string",
"nullable": true
},
"sourceURL": {
"type": "string",
"format": "uri"
},
"<any other metadata> ": {
"type": "string"
},
"pageStatusCode": {
"type": "integer",
"description": "The status code of the page"
@ -647,6 +554,7 @@
"nullable": true,
"description": "The error message of the page"
}
}
},
"llm_extraction": {
@ -694,118 +602,13 @@
"type": "string",
"nullable": true
},
"keywords": {
"type": "string",
"nullable": true
},
"robots": {
"type": "string",
"nullable": true
},
"ogTitle": {
"type": "string",
"nullable": true
},
"ogDescription": {
"type": "string",
"nullable": true
},
"ogUrl": {
"type": "string",
"format": "uri",
"nullable": true
},
"ogImage": {
"type": "string",
"nullable": true
},
"ogAudio": {
"type": "string",
"nullable": true
},
"ogDeterminer": {
"type": "string",
"nullable": true
},
"ogLocale": {
"type": "string",
"nullable": true
},
"ogLocaleAlternate": {
"type": "array",
"items": {
"type": "string"
},
"nullable": true
},
"ogSiteName": {
"type": "string",
"nullable": true
},
"ogVideo": {
"type": "string",
"nullable": true
},
"dctermsCreated": {
"type": "string",
"nullable": true
},
"dcDateCreated": {
"type": "string",
"nullable": true
},
"dcDate": {
"type": "string",
"nullable": true
},
"dctermsType": {
"type": "string",
"nullable": true
},
"dcType": {
"type": "string",
"nullable": true
},
"dctermsAudience": {
"type": "string",
"nullable": true
},
"dctermsSubject": {
"type": "string",
"nullable": true
},
"dcSubject": {
"type": "string",
"nullable": true
},
"dcDescription": {
"type": "string",
"nullable": true
},
"dctermsKeywords": {
"type": "string",
"nullable": true
},
"modifiedTime": {
"type": "string",
"nullable": true
},
"publishedTime": {
"type": "string",
"nullable": true
},
"articleTag": {
"type": "string",
"nullable": true
},
"articleSection": {
"type": "string",
"nullable": true
},
"sourceURL": {
"type": "string",
"format": "uri"
},
"<any other metadata> ": {
"type": "string"
},
"pageStatusCode": {
"type": "integer",
"description": "The status code of the page"

8
apps/api/pnpm-lock.yaml generated
View File

@ -6801,7 +6801,7 @@ packages:
handlebars: 4.7.8
openai: 3.3.0
sbd: 1.0.19
typescript: 5.4.5
typescript: 5.5.3
uuid: 9.0.1
zod: 3.23.8
transitivePeerDependencies:
@ -7767,6 +7767,12 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
/typescript@5.5.3:
resolution: {integrity: sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==}
engines: {node: '>=14.17'}
hasBin: true
dev: false
/typesense@1.8.2(@babel/runtime@7.24.6):
resolution: {integrity: sha512-aBpePjA99Qvo+OP2pJwMpvga4Jrm1Y2oV5NsrWXBxlqUDNEUCPZBIksPv2Hq0jxQxHhLLyJVbjXjByXsvpCDVA==}
engines: {node: '>=18'}

103
apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
View File

@ -132,6 +132,28 @@ describe("E2E Tests for API Routes", () => {
expect(response.body.data.metadata.pageError).toBeUndefined();
}, 30000); // 30 seconds timeout
it.concurrent("should return a successful response with a valid API key and includeRawHtml set to true", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
pageOptions: { includeRawHtml: true },
});
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty("rawHtml");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain("_Roast_");
expect(response.body.data.markdown).toContain("_Roast_");
expect(response.body.data.rawHtml).toContain("<h1");
expect(response.body.data.metadata.pageStatusCode).toBe(200);
expect(response.body.data.metadata.pageError).toBeUndefined();
}, 30000); // 30 seconds timeout
it.concurrent('should return a successful response for a valid scrape with PDF file', async () => {
const response = await request(TEST_URL)
.post('/v0/scrape')
@ -804,6 +826,46 @@ describe("E2E Tests for API Routes", () => {
expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
}, 180000);
it.concurrent("should crawl external content links when allowed", async () => {
const crawlInitResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
crawlerOptions: {
allowExternalContentLinks: true,
ignoreSitemap: true,
returnOnlyUrls: true,
limit: 50
}
});
expect(crawlInitResponse.statusCode).toBe(200);
expect(crawlInitResponse.body).toHaveProperty("jobId");
let crawlStatus: string;
let crawlData = [];
while (crawlStatus !== "completed") {
const statusResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlInitResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
crawlStatus = statusResponse.body.status;
if (statusResponse.body.data) {
crawlData = statusResponse.body.data;
}
if (crawlStatus !== "completed") {
await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
console.log(crawlData)
expect(crawlData.length).toBeGreaterThan(0);
expect(crawlData).toEqual(expect.arrayContaining([
expect.objectContaining({ url: expect.stringContaining("https://firecrawl.dev/?ref=mendable+banner") }),
expect.objectContaining({ url: expect.stringContaining("https://mendable.ai/pricing") }),
expect.objectContaining({ url: expect.stringContaining("https://x.com/CalebPeffer") })
]));
}, 180000); // 3 minutes timeout
});
describe("POST /v0/crawlWebsitePreview", () => {
@ -1177,6 +1239,47 @@ describe("E2E Tests for API Routes", () => {
expect(llmExtraction.is_open_source).toBe(false);
expect(typeof llmExtraction.is_open_source).toBe("boolean");
}, 60000); // 60 secs
it.concurrent("should extract data using LLM extraction mode with RawHtml", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
extractorOptions: {
mode: "llm-extraction-from-raw-html",
extractionPrompt:
"Based on the information on the page, what are the primary and secondary CTA buttons?",
extractionSchema: {
type: "object",
properties: {
primary_cta: {
type: "string",
},
secondary_cta: {
type: "string",
},
},
required: ["primary_cta", "secondary_cta"],
},
},
});
// Ensure that the job was successfully created before proceeding with LLM extraction
expect(response.statusCode).toBe(200);
// Assuming the LLM extraction object is available in the response body under `data.llm_extraction`
let llmExtraction = response.body.data.llm_extraction;
// Check if the llm_extraction object has the required properties with correct types and values
expect(llmExtraction).toHaveProperty("primary_cta");
expect(typeof llmExtraction.primary_cta).toBe("string");
expect(llmExtraction).toHaveProperty("secondary_cta");
expect(typeof llmExtraction.secondary_cta).toBe("string");
}, 60000); // 60 secs
});
// describe("POST /v0/scrape for Top 100 Companies", () => {

77
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -546,46 +546,51 @@ describe("E2E Tests for API Routes", () => {
expect(childrenLinks.length).toBe(completedResponse.body.data.length);
}, 180000); // 120 seconds
it.concurrent('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension ', async () => {
const crawlResponse = await request(TEST_URL)
.post('/v0/crawl')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json')
.send({ url: 'https://arxiv.org/pdf/astro-ph/9301001', crawlerOptions: { limit: 10, excludes: [ 'list/*', 'login', 'abs/*', 'static/*', 'about/*', 'archive/*' ] }});
expect(crawlResponse.statusCode).toBe(200);
// TODO: review the test below
// it.concurrent('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension ', async () => {
// const crawlResponse = await request(TEST_URL)
// .post('/v0/crawl')
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
// .set('Content-Type', 'application/json')
// .send({ url: 'https://arxiv.org/list/astro-ph/1993-01',
// crawlerOptions: {
// limit: 10,
// returnOnlyUrls: true
// }});
// expect(crawlResponse.statusCode).toBe(200);
let isCompleted = false;
let completedResponse;
// let isCompleted = false;
// let completedResponse;
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('status');
// while (!isCompleted) {
// const response = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
// .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty('status');
if (response.body.status === 'completed') {
isCompleted = true;
completedResponse = response;
} else {
await new Promise((r) => setTimeout(r, 1000)); // Wait for 1 second before checking again
}
}
expect(completedResponse.body.status).toBe('completed');
expect(completedResponse.body).toHaveProperty('data');
expect(completedResponse.body.data.length).toEqual(1);
expect(completedResponse.body.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
content: expect.stringContaining('asymmetries might represent, for instance, preferred source orientations to our line of sight.')
})
])
);
// if (response.body.status === 'completed') {
// isCompleted = true;
// completedResponse = response;
// } else {
// await new Promise((r) => setTimeout(r, 1000)); // Wait for 1 second before checking again
// }
// }
// expect(completedResponse.body.status).toBe('completed');
// expect(completedResponse.body).toHaveProperty('data');
// expect(completedResponse.body.data.length).toEqual(1);
// expect(completedResponse.body.data).toEqual(
// expect.arrayContaining([
// expect.objectContaining({
// content: expect.stringContaining('asymmetries might represent, for instance, preferred source orientations to our line of sight.')
// })
// ])
// );
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
}, 180000); // 120 seconds
// expect(completedResponse.body.data[0]).toHaveProperty("metadata");
// expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
// expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
// }, 180000); // 120 seconds
it.concurrent("If someone cancels a crawl job, it should turn into failed status", async () => {
const crawlResponse = await request(TEST_URL)

12
apps/api/src/controllers/scrape.ts
View File

@ -58,19 +58,27 @@ export async function scrapeHelper(
}
// make sure doc.content is not empty
const filteredDocs = docs.filter(
let filteredDocs = docs.filter(
(doc: { content?: string }) => doc.content && doc.content.trim().length > 0
);
if (filteredDocs.length === 0) {
return { success: true, error: "No page found", returnCode: 200, data: docs[0] };
}
// Remove rawHtml if pageOptions.rawHtml is false and extractorOptions.mode is llm-extraction-from-raw-html
if (!pageOptions.includeRawHtml && extractorOptions.mode == "llm-extraction-from-raw-html") {
filteredDocs.forEach(doc => {
delete doc.rawHtml;
});
}
let creditsToBeBilled = filteredDocs.length;
const creditsPerLLMExtract = 50;
if (extractorOptions.mode === "llm-extraction") {
if (extractorOptions.mode === "llm-extraction" || extractorOptions.mode === "llm-extraction-from-raw-html" || extractorOptions.mode === "llm-extraction-from-markdown") {
creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length);
}

4
apps/api/src/lib/LLM-extraction/index.ts
View File

@ -8,7 +8,8 @@ import { Document, ExtractorOptions } from "../entities";
// Generate completion using OpenAI
export async function generateCompletions(
documents: Document[],
extractionOptions: ExtractorOptions
extractionOptions: ExtractorOptions,
mode: "markdown" | "raw-html"
): Promise<Document[]> {
// const schema = zodToJsonSchema(options.schema)
@ -28,6 +29,7 @@ export async function generateCompletions(
document: document,
schema: schema,
prompt: prompt,
mode: mode,
});
// Validate the JSON output against the schema using AJV
const validate = ajv.compile(schema);

27
apps/api/src/lib/LLM-extraction/models.ts
View File

@ -13,26 +13,37 @@ const defaultPrompt =
"You are a professional web scraper. Extract the contents of the webpage";
function prepareOpenAIDoc(
document: Document
document: Document,
mode: "markdown" | "raw-html"
): [OpenAI.Chat.Completions.ChatCompletionContentPart[], number] {
let markdown = document.markdown;
let extractionTarget = document.markdown;
if (mode === "raw-html") {
extractionTarget = document.rawHtml;
}
// Check if the markdown content exists in the document
if (!markdown) {
if (!extractionTarget) {
throw new Error(
"Markdown content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai"
`${mode} content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai`
);
}
// count number of tokens
const numTokens = numTokensFromString(document.markdown, "gpt-4");
const numTokens = numTokensFromString(extractionTarget, "gpt-4");
if (numTokens > maxTokens) {
// trim the document to the maximum number of tokens, tokens != characters
markdown = markdown.slice(0, (maxTokens * modifier));
extractionTarget = extractionTarget.slice(0, (maxTokens * modifier));
}
return [[{ type: "text", text: markdown }], numTokens];
return [[{ type: "text", text: extractionTarget }], numTokens];
}
export async function generateOpenAICompletions({
@ -42,6 +53,7 @@ export async function generateOpenAICompletions({
schema, //TODO - add zod dynamic type checking
prompt = defaultPrompt,
temperature,
mode
}: {
client: OpenAI;
model?: string;
@ -49,9 +61,10 @@ export async function generateOpenAICompletions({
schema: any; // This should be replaced with a proper Zod schema type when available
prompt?: string;
temperature?: number;
mode: "markdown" | "raw-html";
}): Promise<Document> {
const openai = client as OpenAI;
const [content, numTokens] = prepareOpenAIDoc(document);
const [content, numTokens] = prepareOpenAIDoc(document, mode);
const completion = await openai.chat.completions.create({
model,

6
apps/api/src/lib/entities.ts
View File

@ -13,6 +13,7 @@ export interface Progress {
export type PageOptions = {
onlyMainContent?: boolean;
includeHtml?: boolean;
includeRawHtml?: boolean;
fallback?: boolean;
fetchPageContent?: boolean;
waitFor?: number;
@ -21,10 +22,11 @@ export type PageOptions = {
replaceAllPathsWithAbsolutePaths?: boolean;
parsePDF?: boolean;
removeTags?: string | string[];
onlyIncludeTags?: string | string[];
};
export type ExtractorOptions = {
mode: "markdown" | "llm-extraction";
mode: "markdown" | "llm-extraction" | "llm-extraction-from-markdown" | "llm-extraction-from-raw-html";
extractionPrompt?: string;
extractionSchema?: Record<string, any>;
}
@ -50,6 +52,7 @@ export type CrawlerOptions = {
ignoreSitemap?: boolean;
mode?: "default" | "fast"; // have a mode of some sort
allowBackwardCrawling?: boolean;
allowExternalContentLinks?: boolean;
}
export type WebScraperOptions = {
@ -72,6 +75,7 @@ export class Document {
content: string;
markdown?: string;
html?: string;
rawHtml?: string;
llm_extraction?: Record<string, any>;
createdAt?: Date;
updatedAt?: Date;

33
apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
View File

@ -188,5 +188,38 @@ describe('WebCrawler', () => {
// Check that the backward link is included if allowBackwardCrawling is true
expect(results.some(r => r.url === 'https://mendable.ai')).toBe(true);
});
it('should respect the limit parameter by not returning more links than specified', async () => {
const initialUrl = 'http://example.com';
const limit = 2; // Set a limit for the number of links
crawler = new WebCrawler({
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: limit, // Apply the limit
maxCrawledDepth: 10
});
// Mock sitemap fetching function to return more links than the limit
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
initialUrl,
initialUrl + '/page1',
initialUrl + '/page2',
initialUrl + '/page3'
]);
const filteredLinks = crawler['filterLinks'](
[initialUrl, initialUrl + '/page1', initialUrl + '/page2', initialUrl + '/page3'],
limit,
10
);
expect(filteredLinks.length).toBe(limit); // Check if the number of results respects the limit
expect(filteredLinks).toEqual([
initialUrl,
initialUrl + '/page1'
]);
});
});

72
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -4,7 +4,7 @@ import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import async from "async";
import { CrawlerOptions, PageOptions, Progress } from "../../lib/entities";
import { scrapSingleUrl, scrapWithScrapingBee } from "./single_url";
import { scrapSingleUrl } from "./single_url";
import robotsParser from "robots-parser";
import { getURLDepth } from "./utils/maxDepthUtils";
import { axiosTimeout } from "../../../src/lib/timeout";
@ -23,6 +23,7 @@ export class WebCrawler {
private robots: any;
private generateImgAltText: boolean;
private allowBackwardCrawling: boolean;
private allowExternalContentLinks: boolean;
constructor({
initialUrl,
@ -32,7 +33,8 @@ export class WebCrawler {
limit = 10000,
generateImgAltText = false,
maxCrawledDepth = 10,
allowBackwardCrawling = false
allowBackwardCrawling = false,
allowExternalContentLinks = false
}: {
initialUrl: string;
includes?: string[];
@ -42,6 +44,7 @@ export class WebCrawler {
generateImgAltText?: boolean;
maxCrawledDepth?: number;
allowBackwardCrawling?: boolean;
allowExternalContentLinks?: boolean;
}) {
this.initialUrl = initialUrl;
this.baseUrl = new URL(initialUrl).origin;
@ -55,6 +58,7 @@ export class WebCrawler {
this.maxCrawledDepth = maxCrawledDepth ?? 10;
this.generateImgAltText = generateImgAltText ?? false;
this.allowBackwardCrawling = allowBackwardCrawling ?? false;
this.allowExternalContentLinks = allowExternalContentLinks ?? false;
}
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
@ -98,9 +102,10 @@ export class WebCrawler {
const linkHostname = normalizedLink.hostname.replace(/^www\./, '');
// Ensure the protocol and hostname match, and the path starts with the initial URL's path
if (linkHostname !== initialHostname) {
return false;
}
// commented to able to handling external link on allowExternalContentLinks
// if (linkHostname !== initialHostname) {
// return false;
// }
if (!this.allowBackwardCrawling) {
if (!normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)) {
@ -278,16 +283,25 @@ export class WebCrawler {
const path = urlObj.pathname;
if (
this.isInternalLink(fullUrl) &&
if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS
if (this.isInternalLink(fullUrl) &&
this.noSections(fullUrl) &&
// The idea here to comment this out is to allow wider website coverage as we filter this anyway afterwards
// this.matchesIncludes(path) &&
!this.matchesExcludes(path) &&
this.isRobotsAllowed(fullUrl)
) {
links.push({ url: fullUrl, html: content, pageStatusCode, pageError });
}
} else { // EXTERNAL LINKS
if (
this.isInternalLink(url) &&
this.allowExternalContentLinks &&
!this.isSocialMediaOrEmail(fullUrl) &&
!this.matchesExcludes(fullUrl, true) &&
!this.isExternalMainPage(fullUrl)
) {
links.push({ url: fullUrl, html: content, pageStatusCode, pageError });
}
}
}
});
@ -320,9 +334,41 @@ export class WebCrawler {
return this.includes.some((pattern) => new RegExp(pattern).test(url));
}
private matchesExcludes(url: string): boolean {
if (this.excludes.length === 0 || this.excludes[0] == "") return false;
private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {
return this.excludes.some((pattern) => {
if (onlyDomains)
return this.matchesExcludesExternalDomains(url);
return this.excludes.some((pattern) => new RegExp(pattern).test(url));
});
}
// supported formats: "example.com/blog", "https://example.com", "blog.example.com", "example.com"
private matchesExcludesExternalDomains(url: string) {
try {
const urlObj = new URL(url);
const hostname = urlObj.hostname;
const pathname = urlObj.pathname;
for (let domain of this.excludes) {
let domainObj = new URL('http://' + domain.replace(/^https?:\/\//, ''));
let domainHostname = domainObj.hostname;
let domainPathname = domainObj.pathname;
if (hostname === domainHostname || hostname.endsWith(`.${domainHostname}`)) {
if (pathname.startsWith(domainPathname)) {
return true;
}
}
}
return false;
} catch (e) {
return false;
}
}
private isExternalMainPage(url:string):boolean {
return !Boolean(url.split("/").slice(3).filter(subArray => subArray.length > 0).length)
}
private noSections(link: string): boolean {
@ -375,6 +421,10 @@ export class WebCrawler {
"instagram.com",
"pinterest.com",
"mailto:",
"github.com",
"calendly.com",
"discord.gg",
"discord.com",
];
return socialMediaOrEmail.some((ext) => url.includes(ext));
}

19
apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
View File

@ -29,16 +29,16 @@ export async function handleCustomScraping(
};
}
// Check for Google Drive PDF links in the raw HTML
const googleDrivePdfPattern =
/https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/view/;
const googleDrivePdfLink = url.match(googleDrivePdfPattern);
if (googleDrivePdfLink) {
console.log(
`Google Drive PDF link detected for ${url}: ${googleDrivePdfLink[0]}`
);
// Check for Google Drive PDF links in meta tags
const googleDriveMetaPattern = /<meta itemprop="url" content="(https:\/\/drive\.google\.com\/file\/d\/[^"]+)"/;
const googleDriveMetaMatch = text.match(googleDriveMetaPattern);
if (googleDriveMetaMatch) {
const url = googleDriveMetaMatch[1];
console.log(`Google Drive PDF link detected: ${url}`);
const fileId = googleDrivePdfLink[1];
const fileIdMatch = url.match(/https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/view/);
if (fileIdMatch) {
const fileId = fileIdMatch[1];
const pdfUrl = `https://drive.google.com/uc?export=download&id=${fileId}`;
return {
@ -46,6 +46,7 @@ export async function handleCustomScraping(
url: pdfUrl
};
}
}
return null;
}

1
apps/api/src/scraper/WebScraper/global.ts Normal file
View File

@ -0,0 +1 @@
export const universalTimeout = 15000;

14
apps/api/src/scraper/WebScraper/index.ts
View File

@ -40,6 +40,7 @@ export class WebScraperDataProvider {
"gpt-4-turbo";
private crawlerMode: string = "default";
private allowBackwardCrawling: boolean = false;
private allowExternalContentLinks: boolean = false;
authorize(): void {
throw new Error("Method not implemented.");
@ -66,6 +67,7 @@ export class WebScraperDataProvider {
const result = await scrapSingleUrl(
url,
this.pageOptions,
this.extractorOptions,
existingHTML
);
processedUrls++;
@ -172,6 +174,7 @@ export class WebScraperDataProvider {
limit: this.limit,
generateImgAltText: this.generateImgAltText,
allowBackwardCrawling: this.allowBackwardCrawling,
allowExternalContentLinks: this.allowExternalContentLinks,
});
let links = await crawler.start(
@ -269,10 +272,16 @@ export class WebScraperDataProvider {
// documents = await this.applyImgAltText(documents);
if (
this.extractorOptions.mode === "llm-extraction" &&
(this.extractorOptions.mode === "llm-extraction" || this.extractorOptions.mode === "llm-extraction-from-markdown") &&
this.mode === "single_urls"
) {
documents = await generateCompletions(documents, this.extractorOptions);
documents = await generateCompletions(documents, this.extractorOptions, "markdown");
}
if (
(this.extractorOptions.mode === "llm-extraction-from-raw-html") &&
this.mode === "single_urls"
) {
documents = await generateCompletions(documents, this.extractorOptions, "raw-html");
}
return documents.concat(pdfDocuments).concat(docxDocuments);
}
@ -489,6 +498,7 @@ export class WebScraperDataProvider {
this.crawlerMode = options.crawlerOptions?.mode ?? "default";
this.ignoreSitemap = options.crawlerOptions?.ignoreSitemap ?? false;
this.allowBackwardCrawling = options.crawlerOptions?.allowBackwardCrawling ?? false;
this.allowExternalContentLinks = options.crawlerOptions?.allowExternalContentLinks ?? false;
// make sure all urls start with https://
this.urls = this.urls.map((url) => {

77
apps/api/src/scraper/WebScraper/scrapers/fetch.ts Normal file
View File

@ -0,0 +1,77 @@
import axios from "axios";
import { logScrape } from "../../../services/logging/scrape_log";
import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global";
/**
* Scrapes a URL with Axios
* @param url The URL to scrape
* @param pageOptions The options for the page
* @returns The scraped content
*/
export async function scrapWithFetch(
url: string,
pageOptions: { parsePDF?: boolean } = { parsePDF: true }
): Promise<{ content: string; pageStatusCode?: number; pageError?: string }> {
const logParams = {
url,
scraper: "fetch",
success: false,
response_code: null,
time_taken_seconds: null,
error_message: null,
html: "",
startTime: Date.now(),
};
try {
const response = await axios.get(url, {
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout,
transformResponse: [(data) => data], // Prevent axios from parsing JSON automatically
});
if (response.status !== 200) {
console.error(
`[Axios] Error fetching url: ${url} with status: ${response.status}`
);
logParams.error_message = response.statusText;
logParams.response_code = response.status;
return {
content: "",
pageStatusCode: response.status,
pageError: response.statusText,
};
}
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
logParams.success = true;
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
return { content, pageStatusCode, pageError };
} else {
const text = response.data;
logParams.success = true;
logParams.html = text;
logParams.response_code = response.status;
return { content: text, pageStatusCode: response.status, pageError: null };
}
} catch (error) {
if (error.code === "ECONNABORTED") {
logParams.error_message = "Request timed out";
console.log(`[Axios] Request timed out for ${url}`);
} else {
logParams.error_message = error.message || error;
console.error(`[Axios] Error fetching url: ${url} -> ${error}`);
}
return { content: "", pageStatusCode: null, pageError: logParams.error_message };
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
await logScrape(logParams);
}
}

124
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts Normal file
View File

@ -0,0 +1,124 @@
import axios from "axios";
import { FireEngineResponse } from "../../../lib/entities";
import { logScrape } from "../../../services/logging/scrape_log";
import { generateRequestParams } from "../single_url";
import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global";
/**
* Scrapes a URL with Fire-Engine
* @param url The URL to scrape
* @param waitFor The time to wait for the page to load
* @param screenshot Whether to take a screenshot
* @param pageOptions The options for the page
* @param headers The headers to send with the request
* @param options The options for the request
* @returns The scraped content
*/
export async function scrapWithFireEngine({
url,
waitFor = 0,
screenshot = false,
pageOptions = { parsePDF: true },
headers,
options,
}: {
url: string;
waitFor?: number;
screenshot?: boolean;
pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean };
headers?: Record<string, string>;
options?: any;
}): Promise<FireEngineResponse> {
const logParams = {
url,
scraper: "fire-engine",
success: false,
response_code: null,
time_taken_seconds: null,
error_message: null,
html: "",
startTime: Date.now(),
};
try {
const reqParams = await generateRequestParams(url);
const waitParam = reqParams["params"]?.wait ?? waitFor;
const screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
console.log(
`[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam}`
);
const response = await axios.post(
process.env.FIRE_ENGINE_BETA_URL + "/scrape",
{
url: url,
wait: waitParam,
screenshot: screenshotParam,
headers: headers,
pageOptions: pageOptions,
},
{
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout + waitParam,
}
);
if (response.status !== 200) {
console.error(
`[Fire-Engine] Error fetching url: ${url} with status: ${response.status}`
);
logParams.error_message = response.data?.pageError;
logParams.response_code = response.data?.pageStatusCode;
return {
html: "",
screenshot: "",
pageStatusCode: response.data?.pageStatusCode,
pageError: response.data?.pageError,
};
}
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
url,
pageOptions?.parsePDF
);
logParams.success = true;
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
return { html: content, screenshot: "", pageStatusCode, pageError };
} else {
const data = response.data;
logParams.success =
(data.pageStatusCode >= 200 && data.pageStatusCode < 300) ||
data.pageStatusCode === 404;
logParams.html = data.content ?? "";
logParams.response_code = data.pageStatusCode;
logParams.error_message = data.pageError;
return {
html: data.content ?? "",
screenshot: data.screenshot ?? "",
pageStatusCode: data.pageStatusCode,
pageError: data.pageError,
};
}
} catch (error) {
if (error.code === "ECONNABORTED") {
console.log(`[Fire-Engine] Request timed out for ${url}`);
logParams.error_message = "Request timed out";
} else {
console.error(`[Fire-Engine][c] Error fetching url: ${url} -> ${error}`);
logParams.error_message = error.message || error;
}
return { html: "", screenshot: "", pageStatusCode: null, pageError: logParams.error_message };
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
await logScrape(logParams, pageOptions);
}
}

109
apps/api/src/scraper/WebScraper/scrapers/playwright.ts Normal file
View File

@ -0,0 +1,109 @@
import axios from "axios";
import { logScrape } from "../../../services/logging/scrape_log";
import { generateRequestParams } from "../single_url";
import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global";
/**
* Scrapes a URL with Playwright
* @param url The URL to scrape
* @param waitFor The time to wait for the page to load
* @param headers The headers to send with the request
* @param pageOptions The options for the page
* @returns The scraped content
*/
export async function scrapWithPlaywright(
url: string,
waitFor: number = 0,
headers?: Record<string, string>,
pageOptions: { parsePDF?: boolean } = { parsePDF: true }
): Promise<{ content: string; pageStatusCode?: number; pageError?: string }> {
const logParams = {
url,
scraper: "playwright",
success: false,
response_code: null,
time_taken_seconds: null,
error_message: null,
html: "",
startTime: Date.now(),
};
try {
const reqParams = await generateRequestParams(url);
// If the user has passed a wait parameter in the request, use that
const waitParam = reqParams["params"]?.wait ?? waitFor;
const response = await axios.post(
process.env.PLAYWRIGHT_MICROSERVICE_URL,
{
url: url,
wait_after_load: waitParam,
headers: headers,
},
{
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout + waitParam, // Add waitParam to timeout to account for the wait time
transformResponse: [(data) => data], // Prevent axios from parsing JSON automatically
}
);
if (response.status !== 200) {
console.error(
`[Playwright] Error fetching url: ${url} with status: ${response.status}`
);
logParams.error_message = response.data?.pageError;
logParams.response_code = response.data?.pageStatusCode;
return {
content: "",
pageStatusCode: response.data?.pageStatusCode,
pageError: response.data?.pageError,
};
}
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
logParams.success = true;
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
return { content, pageStatusCode, pageError };
} else {
const textData = response.data;
try {
const data = JSON.parse(textData);
const html = data.content;
logParams.success = true;
logParams.html = html;
logParams.response_code = data.pageStatusCode;
logParams.error_message = data.pageError;
return {
content: html ?? "",
pageStatusCode: data.pageStatusCode,
pageError: data.pageError,
};
} catch (jsonError) {
logParams.error_message = jsonError.message || jsonError;
console.error(
`[Playwright] Error parsing JSON response for url: ${url} -> ${jsonError}`
);
return { content: "", pageStatusCode: null, pageError: logParams.error_message };
}
}
} catch (error) {
if (error.code === "ECONNABORTED") {
logParams.error_message = "Request timed out";
console.log(`[Playwright] Request timed out for ${url}`);
} else {
logParams.error_message = error.message || error;
console.error(`[Playwright] Error fetching url: ${url} -> ${error}`);
}
return { content: "", pageStatusCode: null, pageError: logParams.error_message };
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
await logScrape(logParams);
}
}

88
apps/api/src/scraper/WebScraper/scrapers/scrapingBee.ts Normal file
View File

@ -0,0 +1,88 @@
import { logScrape } from "../../../services/logging/scrape_log";
import { generateRequestParams } from "../single_url";
import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global";
import { ScrapingBeeClient } from "scrapingbee";
/**
* Scrapes a URL with ScrapingBee
* @param url The URL to scrape
* @param wait_browser The browser event to wait for
* @param timeout The timeout for the scrape
* @param pageOptions The options for the page
* @returns The scraped content
*/
export async function scrapWithScrapingBee(
url: string,
wait_browser: string = "domcontentloaded",
timeout: number = universalTimeout,
pageOptions: { parsePDF?: boolean } = { parsePDF: true }
): Promise<{ content: string; pageStatusCode?: number; pageError?: string }> {
const logParams = {
url,
scraper: wait_browser === "networkidle2" ? "scrapingBeeLoad" : "scrapingBee",
success: false,
response_code: null,
time_taken_seconds: null,
error_message: null,
html: "",
startTime: Date.now(),
};
try {
const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY);
const clientParams = await generateRequestParams(
url,
wait_browser,
timeout
);
const response = await client.get({
...clientParams,
params: {
...clientParams.params,
transparent_status_code: "True",
},
});
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
logParams.success = true;
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
return { content, pageStatusCode, pageError };
} else {
let text = "";
try {
const decoder = new TextDecoder();
text = decoder.decode(response.data);
logParams.success = true;
} catch (decodeError) {
console.error(
`[ScrapingBee][c] Error decoding response data for url: ${url} -> ${decodeError}`
);
logParams.error_message = decodeError.message || decodeError;
}
logParams.response_code = response.status;
logParams.html = text;
logParams.success = response.status >= 200 && response.status < 300 || response.status === 404;
logParams.error_message = response.statusText !== "OK" ? response.statusText : undefined;
return {
content: text,
pageStatusCode: response.status,
pageError: response.statusText !== "OK" ? response.statusText : undefined,
};
}
} catch (error) {
console.error(`[ScrapingBee][c] Error fetching url: ${url} -> ${error}`);
logParams.error_message = error.message || error;
logParams.response_code = error.response?.status;
return {
content: "",
pageStatusCode: error.response?.status,
pageError: error.response?.statusText,
};
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;
await logScrape(logParams);
}
}

304
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -1,14 +1,21 @@
import * as cheerio from "cheerio";
import { ScrapingBeeClient } from "scrapingbee";
import { extractMetadata } from "./utils/metadata";
import dotenv from "dotenv";
import { Document, PageOptions, FireEngineResponse } from "../../lib/entities";
import {
Document,
PageOptions,
FireEngineResponse,
ExtractorOptions,
} from "../../lib/entities";
import { parseMarkdown } from "../../lib/html-to-markdown";
import { urlSpecificParams } from "./utils/custom/website_params";
import { fetchAndProcessPdf } from "./utils/pdfProcessor";
import { handleCustomScraping } from "./custom/handleCustomScraping";
import { removeUnwantedElements } from "./utils/removeUnwantedElements";
import axios from "axios";
import { scrapWithFetch } from "./scrapers/fetch";
import { scrapWithFireEngine } from "./scrapers/fireEngine";
import { scrapWithPlaywright } from "./scrapers/playwright";
import { scrapWithScrapingBee } from "./scrapers/scrapingBee";
dotenv.config();
@ -20,8 +27,6 @@ const baseScrapers = [
"fetch",
] as const;
const universalTimeout = 15000;
export async function generateRequestParams(
url: string,
wait_browser: string = "domcontentloaded",
@ -45,197 +50,6 @@ export async function generateRequestParams(
return defaultParams;
}
}
export async function scrapWithFireEngine(
url: string,
waitFor: number = 0,
screenshot: boolean = false,
pageOptions: { scrollXPaths?: string[], parsePDF?: boolean } = { parsePDF: true },
headers?: Record<string, string>,
options?: any
): Promise<FireEngineResponse> {
try {
const reqParams = await generateRequestParams(url);
// If the user has passed a wait parameter in the request, use that
const waitParam = reqParams["params"]?.wait ?? waitFor;
const screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
console.log(
`[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam}`
);
const response = await axios.post(
process.env.FIRE_ENGINE_BETA_URL + "/scrape",
{
url: url,
wait: waitParam,
screenshot: screenshotParam,
headers: headers,
pageOptions: pageOptions,
},
{
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout + waitParam
}
);
if (response.status !== 200) {
console.error(
`[Fire-Engine] Error fetching url: ${url} with status: ${response.status}`
);
return { html: "", screenshot: "", pageStatusCode: response.data?.pageStatusCode, pageError: response.data?.pageError };
}
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
return { html: content, screenshot: "", pageStatusCode, pageError };
} else {
const data = response.data;
const html = data.content;
const screenshot = data.screenshot;
return { html: html ?? "", screenshot: screenshot ?? "", pageStatusCode: data.pageStatusCode, pageError: data.pageError };
}
} catch (error) {
if (error.code === 'ECONNABORTED') {
console.log(`[Fire-Engine] Request timed out for ${url}`);
} else {
console.error(`[Fire-Engine][c] Error fetching url: ${url} -> ${error}`);
}
return { html: "", screenshot: "" };
}
}
export async function scrapWithScrapingBee(
url: string,
wait_browser: string = "domcontentloaded",
timeout: number = universalTimeout,
pageOptions: { parsePDF?: boolean } = { parsePDF: true }
): Promise<{ content: string, pageStatusCode?: number, pageError?: string }> {
try {
const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY);
const clientParams = await generateRequestParams(
url,
wait_browser,
timeout,
);
const response = await client.get({
...clientParams,
params: {
...clientParams.params,
'transparent_status_code': 'True'
}
});
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
return await fetchAndProcessPdf(url, pageOptions?.parsePDF);
} else {
let text = "";
try {
const decoder = new TextDecoder();
text = decoder.decode(response.data);
} catch (decodeError) {
console.error(`[ScrapingBee][c] Error decoding response data for url: ${url} -> ${decodeError}`);
}
return { content: text, pageStatusCode: response.status, pageError: response.statusText != "OK" ? response.statusText : undefined };
}
} catch (error) {
console.error(`[ScrapingBee][c] Error fetching url: ${url} -> ${error}`);
return { content: "", pageStatusCode: error.response.status, pageError: error.response.statusText };
}
}
export async function scrapWithPlaywright(
url: string,
waitFor: number = 0,
headers?: Record<string, string>,
pageOptions: { parsePDF?: boolean } = { parsePDF: true }
): Promise<{ content: string, pageStatusCode?: number, pageError?: string }> {
try {
const reqParams = await generateRequestParams(url);
// If the user has passed a wait parameter in the request, use that
const waitParam = reqParams["params"]?.wait ?? waitFor;
const response = await axios.post(process.env.PLAYWRIGHT_MICROSERVICE_URL, {
url: url,
wait_after_load: waitParam,
headers: headers,
}, {
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout + waitParam, // Add waitParam to timeout to account for the wait time
transformResponse: [(data) => data] // Prevent axios from parsing JSON automatically
});
if (response.status !== 200) {
console.error(
`[Playwright] Error fetching url: ${url} with status: ${response.status}`
);
return { content: "", pageStatusCode: response.data?.pageStatusCode, pageError: response.data?.pageError };
}
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
return await fetchAndProcessPdf(url, pageOptions?.parsePDF);
} else {
const textData = response.data;
try {
const data = JSON.parse(textData);
const html = data.content;
return { content: html ?? "", pageStatusCode: data.pageStatusCode, pageError: data.pageError };
} catch (jsonError) {
console.error(`[Playwright] Error parsing JSON response for url: ${url} -> ${jsonError}`);
return { content: "" };
}
}
} catch (error) {
if (error.code === 'ECONNABORTED') {
console.log(`[Playwright] Request timed out for ${url}`);
} else {
console.error(`[Playwright] Error fetching url: ${url} -> ${error}`);
}
return { content: "" };
}
}
export async function scrapWithFetch(
url: string,
pageOptions: { parsePDF?: boolean } = { parsePDF: true }
): Promise<{ content: string, pageStatusCode?: number, pageError?: string }> {
try {
const response = await axios.get(url, {
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout,
transformResponse: [(data) => data] // Prevent axios from parsing JSON automatically
});
if (response.status !== 200) {
console.error(
`[Axios] Error fetching url: ${url} with status: ${response.status}`
);
return { content: "", pageStatusCode: response.status, pageError: response.statusText };
}
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
return await fetchAndProcessPdf(url, pageOptions?.parsePDF);
} else {
const text = response.data;
return { content: text, pageStatusCode: 200 };
}
} catch (error) {
if (error.code === 'ECONNABORTED') {
console.log(`[Axios] Request timed out for ${url}`);
} else {
console.error(`[Axios] Error fetching url: ${url} -> ${error}`);
}
return { content: "" };
}
}
/**
* Get the order of scrapers to be used for scraping a URL
@ -295,17 +109,18 @@ function getScrapingFallbackOrder(
return scrapersInOrder as (typeof baseScrapers)[number][];
}
export async function scrapSingleUrl(
urlToScrap: string,
pageOptions: PageOptions = {
onlyMainContent: true,
includeHtml: false,
includeRawHtml: false,
waitFor: 0,
screenshot: false,
headers: undefined
headers: undefined,
},
extractorOptions: ExtractorOptions = {
mode: "llm-extraction-from-markdown",
},
existingHtml: string = ""
): Promise<Document> {
@ -315,18 +130,23 @@ export async function scrapSingleUrl(
url: string,
method: (typeof baseScrapers)[number]
) => {
let scraperResponse: { text: string, screenshot: string, metadata: { pageStatusCode?: number, pageError?: string | null } } = { text: "", screenshot: "", metadata: {} };
let scraperResponse: {
text: string;
screenshot: string;
metadata: { pageStatusCode?: number; pageError?: string | null };
} = { text: "", screenshot: "", metadata: {} };
let screenshot = "";
switch (method) {
case "fire-engine":
if (process.env.FIRE_ENGINE_BETA_URL) {
console.log(`Scraping ${url} with Fire Engine`);
const response = await scrapWithFireEngine(
const response = await scrapWithFireEngine({
url,
pageOptions.waitFor,
pageOptions.screenshot,
pageOptions.headers
);
waitFor: pageOptions.waitFor,
screenshot: pageOptions.screenshot,
pageOptions: pageOptions,
headers: pageOptions.headers,
});
scraperResponse.text = response.html;
scraperResponse.screenshot = response.screenshot;
scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
@ -347,7 +167,11 @@ export async function scrapSingleUrl(
break;
case "playwright":
if (process.env.PLAYWRIGHT_MICROSERVICE_URL) {
const response = await scrapWithPlaywright(url, pageOptions.waitFor, pageOptions.headers);
const response = await scrapWithPlaywright(
url,
pageOptions.waitFor,
pageOptions.headers
);
scraperResponse.text = response.content;
scraperResponse.metadata.pageStatusCode = response.pageStatusCode;
scraperResponse.metadata.pageError = response.pageError;
@ -372,19 +196,36 @@ export async function scrapSingleUrl(
let customScrapedContent: FireEngineResponse | null = null;
// Check for custom scraping conditions
const customScraperResult = await handleCustomScraping(scraperResponse.text, url);
const customScraperResult = await handleCustomScraping(
scraperResponse.text,
url
);
if (customScraperResult) {
switch (customScraperResult.scraper) {
case "fire-engine":
customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions)
customScrapedContent = await scrapWithFireEngine({
url: customScraperResult.url,
waitFor: customScraperResult.waitAfterLoad,
screenshot: false,
pageOptions: customScraperResult.pageOptions,
});
if (screenshot) {
customScrapedContent.screenshot = screenshot;
}
break;
case "pdf":
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(customScraperResult.url, pageOptions?.parsePDF);
customScrapedContent = { html: content, screenshot, pageStatusCode, pageError }
const { content, pageStatusCode, pageError } =
await fetchAndProcessPdf(
customScraperResult.url,
pageOptions?.parsePDF
);
customScrapedContent = {
html: content,
screenshot,
pageStatusCode,
pageError,
};
break;
}
}
@ -402,11 +243,18 @@ export async function scrapSingleUrl(
rawHtml: scraperResponse.text,
screenshot: scraperResponse.screenshot,
pageStatusCode: scraperResponse.metadata.pageStatusCode,
pageError: scraperResponse.metadata.pageError || undefined
pageError: scraperResponse.metadata.pageError || undefined,
};
};
let { text, html, rawHtml, screenshot, pageStatusCode, pageError } = { text: "", html: "", rawHtml: "", screenshot: "", pageStatusCode: 200, pageError: undefined };
let { text, html, rawHtml, screenshot, pageStatusCode, pageError } = {
text: "",
html: "",
rawHtml: "",
screenshot: "",
pageStatusCode: 200,
pageError: undefined,
};
try {
let urlKey = urlToScrap;
try {
@ -432,18 +280,20 @@ export async function scrapSingleUrl(
}
const attempt = await attemptScraping(urlToScrap, scraper);
text = attempt.text ?? '';
html = attempt.html ?? '';
rawHtml = attempt.rawHtml ?? '';
screenshot = attempt.screenshot ?? '';
text = attempt.text ?? "";
html = attempt.html ?? "";
rawHtml = attempt.rawHtml ?? "";
screenshot = attempt.screenshot ?? "";
if (attempt.pageStatusCode) {
pageStatusCode = attempt.pageStatusCode;
}
if (attempt.pageError) {
if (attempt.pageError && attempt.pageStatusCode >= 400) {
pageError = attempt.pageError;
} else if (attempt && attempt.pageStatusCode && attempt.pageStatusCode < 400) {
pageError = undefined;
}
if (text && text.trim().length >= 100) break;
if (pageStatusCode && pageStatusCode == 404) break;
const nextScraperIndex = scrapersInOrder.indexOf(scraper) + 1;
@ -465,12 +315,17 @@ export async function scrapSingleUrl(
content: text,
markdown: text,
html: pageOptions.includeHtml ? html : undefined,
rawHtml:
pageOptions.includeRawHtml ||
extractorOptions.mode === "llm-extraction-from-raw-html"
? rawHtml
: undefined,
metadata: {
...metadata,
screenshot: screenshot,
sourceURL: urlToScrap,
pageStatusCode: pageStatusCode,
pageError: pageError
pageError: pageError,
},
};
} else {
@ -478,11 +333,16 @@ export async function scrapSingleUrl(
content: text,
markdown: text,
html: pageOptions.includeHtml ? html : undefined,
rawHtml:
pageOptions.includeRawHtml ||
extractorOptions.mode === "llm-extraction-from-raw-html"
? rawHtml
: undefined,
metadata: {
...metadata,
sourceURL: urlToScrap,
pageStatusCode: pageStatusCode,
pageError: pageError
pageError: pageError,
},
};
}
@ -497,7 +357,7 @@ export async function scrapSingleUrl(
metadata: {
sourceURL: urlToScrap,
pageStatusCode: pageStatusCode,
pageError: pageError
pageError: pageError,
},
} as Document;
}

72
apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
View File

@ -100,4 +100,76 @@ describe('removeUnwantedElements', () => {
expect(result).not.toContain('id="remove-this"');
expect(result).toContain('class="keep"');
});
it('should only include specified tags', () => {
const html = `<div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: ['main', 'footer'] };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<main>Main Content</main>');
expect(result).toContain('<footer>Footer Content</footer>');
expect(result).not.toContain('<aside>');
});
it('should handle multiple specified tags', () => {
const html = `<div><header>Header Content</header><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: ['header', 'main', 'footer'] };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<header>Header Content</header>');
expect(result).toContain('<main>Main Content</main>');
expect(result).toContain('<footer>Footer Content</footer>');
expect(result).not.toContain('<aside>');
});
it('should handle nested specified tags', () => {
const html = `<div><main><section>Main Section</section></main><aside>Remove</aside><footer>Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: ['main', 'footer'] };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<main><section>Main Section</section></main>');
expect(result).toContain('<footer>Footer Content</footer>');
expect(result).not.toContain('<aside>');
});
it('should not handle no specified tags, return full content', () => {
const html = `<html><body><div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div></body></html>`;
const options: PageOptions = { onlyIncludeTags: [] };
const result = removeUnwantedElements(html, options);
expect(result).toBe(html);
});
it('should handle specified tags as a string', () => {
const html = `<div><main>Main Content</main><aside>Remove</aside><footer>Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: 'main' };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<main>Main Content</main>');
expect(result).not.toContain('<aside>');
expect(result).not.toContain('<footer>');
});
it('should include specified tags with class', () => {
const html = `<div><main class="main-content">Main Content</main><aside class="remove">Remove</aside><footer class="footer-content">Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: ['.main-content', '.footer-content'] };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<main class="main-content">Main Content</main>');
expect(result).toContain('<footer class="footer-content">Footer Content</footer>');
expect(result).not.toContain('<aside class="remove">');
});
it('should include specified tags with id', () => {
const html = `<div><main id="main-content">Main Content</main><aside id="remove">Remove</aside><footer id="footer-content">Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: ['#main-content', '#footer-content'] };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<main id="main-content">Main Content</main>');
expect(result).toContain('<footer id="footer-content">Footer Content</footer>');
expect(result).not.toContain('<aside id="remove">');
});
it('should include specified tags with mixed class and id', () => {
const html = `<div><main class="main-content">Main Content</main><aside id="remove">Remove</aside><footer id="footer-content">Footer Content</footer></div>`;
const options: PageOptions = { onlyIncludeTags: ['.main-content', '#footer-content'] };
const result = removeUnwantedElements(html, options);
expect(result).toContain('<main class="main-content">Main Content</main>');
expect(result).toContain('<footer id="footer-content">Footer Content</footer>');
expect(result).not.toContain('<aside id="remove">');
});
});

36
apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
View File

@ -2,12 +2,32 @@ import cheerio, { AnyNode, Cheerio } from "cheerio";
import { PageOptions } from "../../../lib/entities";
import { excludeNonMainTags } from "./excludeTags";
export const removeUnwantedElements = (html: string, pageOptions: PageOptions) => {
export const removeUnwantedElements = (
html: string,
pageOptions: PageOptions
) => {
const soup = cheerio.load(html);
if (pageOptions.onlyIncludeTags) {
if (typeof pageOptions.onlyIncludeTags === "string") {
pageOptions.onlyIncludeTags = [pageOptions.onlyIncludeTags];
}
if (pageOptions.onlyIncludeTags.length !== 0) {
// Create a new root element to hold the tags to keep
const newRoot = cheerio.load("<div></div>")("div");
pageOptions.onlyIncludeTags.forEach((tag) => {
soup(tag).each((index, element) => {
newRoot.append(soup(element).clone());
});
});
return newRoot.html();
}
}
soup("script, style, iframe, noscript, meta, head").remove();
if (pageOptions.removeTags) {
if (typeof pageOptions.removeTags === 'string') {
if (typeof pageOptions.removeTags === "string") {
pageOptions.removeTags = [pageOptions.removeTags];
}
@ -17,16 +37,16 @@ export const removeUnwantedElements = (html: string, pageOptions: PageOptions) =
if (tag.startsWith("*") && tag.endsWith("*")) {
let classMatch = false;
const regexPattern = new RegExp(tag.slice(1, -1), 'i');
elementsToRemove = soup('*').filter((i, element) => {
if (element.type === 'tag') {
const regexPattern = new RegExp(tag.slice(1, -1), "i");
elementsToRemove = soup("*").filter((i, element) => {
if (element.type === "tag") {
const attributes = element.attribs;
const tagNameMatches = regexPattern.test(element.name);
const attributesMatch = Object.keys(attributes).some(attr =>
const attributesMatch = Object.keys(attributes).some((attr) =>
regexPattern.test(`${attr}="${attributes[attr]}"`)
);
if (tag.startsWith('*.')) {
classMatch = Object.keys(attributes).some(attr =>
if (tag.startsWith("*.")) {
classMatch = Object.keys(attributes).some((attr) =>
regexPattern.test(`class="${attributes[attr]}"`)
);
}

21
apps/api/src/services/logging/log_job.ts
View File

@ -1,4 +1,4 @@
import { ExtractorOptions } from './../../lib/entities';
import { ExtractorOptions } from "./../../lib/entities";
import { supabase_service } from "../supabase";
import { FirecrawlJob } from "../../types";
import { posthog } from "../posthog";
@ -10,6 +10,16 @@ export async function logJob(job: FirecrawlJob) {
return;
}
// Redact any pages that have an authorization header
if (
job.pageOptions &&
job.pageOptions.headers &&
job.pageOptions.headers["Authorization"]
) {
job.pageOptions.headers["Authorization"] = "REDACTED";
job.docs = [{ content: "REDACTED DUE TO AUTHORIZATION HEADER", html: "REDACTED DUE TO AUTHORIZATION HEADER" }];
}
const { data, error } = await supabase_service
.from("firecrawl_jobs")
.insert([
@ -27,16 +37,15 @@ export async function logJob(job: FirecrawlJob) {
page_options: job.pageOptions,
origin: job.origin,
extractor_options: job.extractor_options,
num_tokens: job.num_tokens
num_tokens: job.num_tokens,
},
]);
if (process.env.POSTHOG_API_KEY) {
let phLog = {
distinctId: "from-api", //* To identify this on the group level, setting distinctid to a static string per posthog docs: https://posthog.com/docs/product-analytics/group-analytics#advanced-server-side-only-capturing-group-events-without-a-user
...(job.team_id !== "preview" && {
groups: { team: job.team_id }
groups: { team: job.team_id },
}), //* Identifying event on this team
event: "job-logged",
properties: {
@ -51,9 +60,9 @@ export async function logJob(job: FirecrawlJob) {
page_options: job.pageOptions,
origin: job.origin,
extractor_options: job.extractor_options,
num_tokens: job.num_tokens
num_tokens: job.num_tokens,
},
}
};
posthog.capture(phLog);
}
if (error) {

47
apps/api/src/services/logging/scrape_log.ts Normal file
View File

@ -0,0 +1,47 @@
import "dotenv/config";
import { ScrapeLog } from "../../types";
import { supabase_service } from "../supabase";
import { PageOptions } from "../../lib/entities";
export async function logScrape(
scrapeLog: ScrapeLog,
pageOptions?: PageOptions
) {
try {
// Only log jobs in production
// if (process.env.ENV !== "production") {
// return;
// }
// Redact any pages that have an authorization header
if (
pageOptions &&
pageOptions.headers &&
pageOptions.headers["Authorization"]
) {
scrapeLog.html = "REDACTED DUE TO AUTHORIZATION HEADER";
}
const { data, error } = await supabase_service.from("scrape_logs").insert([
{
url: scrapeLog.url,
scraper: scrapeLog.scraper,
success: scrapeLog.success,
response_code: scrapeLog.response_code,
time_taken_seconds: scrapeLog.time_taken_seconds,
proxy: scrapeLog.proxy,
retried: scrapeLog.retried,
error_message: scrapeLog.error_message,
date_added: new Date().toISOString(),
html: scrapeLog.html,
ipv4_support: scrapeLog.ipv4_support,
ipv6_support: scrapeLog.ipv6_support,
},
]);
if (error) {
console.error("Error logging proxy:\n", error);
}
} catch (error) {
console.error("Error logging proxy:\n", error);
}
}

15
apps/api/src/types.ts
View File

@ -115,3 +115,18 @@ export enum NotificationType {
LIMIT_REACHED = "limitReached",
RATE_LIMIT_REACHED = "rateLimitReached",
}
export type ScrapeLog = {
url: string;
scraper: string;
success?: boolean;
response_code?: number;
time_taken_seconds?: number;
proxy?: string;
retried?: boolean;
error_message?: string;
date_added?: string; // ISO 8601 format
html?: string;
ipv4_support?: boolean | null;
ipv6_support?: boolean | null;
};

2
apps/js-sdk/firecrawl/types/index.d.ts vendored
View File

@ -107,7 +107,7 @@ export interface Params {
[key: string]: any;
extractorOptions?: {
extractionSchema: z.ZodSchema | any;
mode?: "llm-extraction";
mode?: "llm-extraction" | "llm-extraction-from-raw-html";
extractionPrompt?: string;
};
}

19
apps/playwright-service-ts/Dockerfile Normal file
View File

@ -0,0 +1,19 @@
FROM node:18-slim
WORKDIR /usr/src/app
COPY package*.json ./
RUN npm install
COPY . .
# Install Playwright dependencies
RUN npx playwright install --with-deps
RUN npm run build
ARG PORT
ENV PORT=${PORT}
EXPOSE ${PORT}
CMD [ "npm", "start" ]

47
apps/playwright-service-ts/README.md Normal file
View File

@ -0,0 +1,47 @@
# Playwright Scrape API
This is a simple web scraping service built with Express and Playwright.
## Features
- Scrapes HTML content from specified URLs.
- Blocks requests to known ad-serving domains.
- Blocks media files to reduce bandwidth usage.
- Uses random user-agent strings to avoid detection.
- Strategy to ensure the page is fully rendered.
## Install
```bash
npm install
npx playwright install
```
## RUN
```bash
npm run build
npm start
```
OR
```bash
npm run dev
```
## USE
```bash
curl -X POST http://localhost:3000/scrape \
-H "Content-Type: application/json" \
-d '{
"url": "https://example.com",
"wait_after_load": 1000,
"timeout": 15000,
"headers": {
"Custom-Header": "value"
},
"check_selector": "#content"
}'
```
## USING WITH FIRECRAWL
Add `PLAYWRIGHT_MICROSERVICE_URL=http://localhost:3003/scrape` to `/apps/api/.env` to configure the API to use this Playwright microservice for scraping operations.

227
apps/playwright-service-ts/api.ts Normal file
View File

@ -0,0 +1,227 @@
import express, { Request, Response } from 'express';
import bodyParser from 'body-parser';
import { chromium, Browser, BrowserContext, Route, Request as PlaywrightRequest } from 'playwright';
import dotenv from 'dotenv';
import randomUseragent from 'random-useragent';
import { getError } from './helpers/get_error';
dotenv.config();
const app = express();
const port = process.env.PORT || 3003;
app.use(bodyParser.json());
const BLOCK_MEDIA = (process.env.BLOCK_MEDIA || 'False').toUpperCase() === 'TRUE';
const PROXY_SERVER = process.env.PROXY_SERVER || null;
const PROXY_USERNAME = process.env.PROXY_USERNAME || null;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD || null;
const AD_SERVING_DOMAINS = [
'doubleclick.net',
'adservice.google.com',
'googlesyndication.com',
'googletagservices.com',
'googletagmanager.com',
'google-analytics.com',
'adsystem.com',
'adservice.com',
'adnxs.com',
'ads-twitter.com',
'facebook.net',
'fbcdn.net',
'amazon-adsystem.com'
];
interface UrlModel {
url: string;
wait_after_load?: number;
timeout?: number;
headers?: { [key: string]: string };
check_selector?: string;
}
let browser: Browser;
let context: BrowserContext;
const initializeBrowser = async () => {
browser = await chromium.launch({
headless: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--no-first-run',
'--no-zygote',
'--single-process',
'--disable-gpu'
]
});
const userAgent = randomUseragent.getRandom();
const viewport = { width: 1280, height: 800 };
const contextOptions: any = {
userAgent,
viewport,
};
if (PROXY_SERVER && PROXY_USERNAME && PROXY_PASSWORD) {
contextOptions.proxy = {
server: PROXY_SERVER,
username: PROXY_USERNAME,
password: PROXY_PASSWORD,
};
} else if (PROXY_SERVER) {
contextOptions.proxy = {
server: PROXY_SERVER,
};
}
context = await browser.newContext(contextOptions);
if (BLOCK_MEDIA) {
await context.route('**/*.{png,jpg,jpeg,gif,svg,mp3,mp4,avi,flac,ogg,wav,webm}', async (route: Route, request: PlaywrightRequest) => {
await route.abort();
});
}
// Intercept all requests to avoid loading ads
await context.route('**/*', (route: Route, request: PlaywrightRequest) => {
const requestUrl = new URL(request.url());
const hostname = requestUrl.hostname;
if (AD_SERVING_DOMAINS.some(domain => hostname.includes(domain))) {
console.log(hostname);
return route.abort();
}
return route.continue();
});
};
const shutdownBrowser = async () => {
if (context) {
await context.close();
}
if (browser) {
await browser.close();
}
};
const isValidUrl = (urlString: string): boolean => {
try {
new URL(urlString);
return true;
} catch (_) {
return false;
}
};
const scrapePage = async (page: any, url: string, waitUntil: 'load' | 'networkidle', waitAfterLoad: number, timeout: number, checkSelector: string | undefined) => {
console.log(`Navigating to ${url} with waitUntil: ${waitUntil} and timeout: ${timeout}ms`);
const response = await page.goto(url, { waitUntil, timeout });
if (waitAfterLoad > 0) {
await page.waitForTimeout(waitAfterLoad);
}
if (checkSelector) {
try {
await page.waitForSelector(checkSelector, { timeout });
} catch (error) {
throw new Error('Required selector not found');
}
}
return {
content: await page.content(),
status: response ? response.status() : null,
};
};
app.post('/scrape', async (req: Request, res: Response) => {
const { url, wait_after_load = 0, timeout = 15000, headers, check_selector }: UrlModel = req.body;
console.log(`================= Scrape Request =================`);
console.log(`URL: ${url}`);
console.log(`Wait After Load: ${wait_after_load}`);
console.log(`Timeout: ${timeout}`);
console.log(`Headers: ${headers ? JSON.stringify(headers) : 'None'}`);
console.log(`Check Selector: ${check_selector ? check_selector : 'None'}`);
console.log(`==================================================`);
if (!url) {
return res.status(400).json({ error: 'URL is required' });
}
if (!isValidUrl(url)) {
return res.status(400).json({ error: 'Invalid URL' });
}
if (!PROXY_SERVER) {
console.warn('⚠️ WARNING: No proxy server provided. Your IP address may be blocked.');
}
if (!browser || !context) {
await initializeBrowser();
}
const page = await context.newPage();
// Set headers if provided
if (headers) {
await page.setExtraHTTPHeaders(headers);
}
let pageContent;
let pageStatusCode: number | null = null;
try {
// Strategy 1: Normal
console.log('Attempting strategy 1: Normal load');
const result = await scrapePage(page, url, 'load', wait_after_load, timeout, check_selector);
pageContent = result.content;
pageStatusCode = result.status;
} catch (error) {
console.log('Strategy 1 failed, attempting strategy 2: Wait until networkidle');
try {
// Strategy 2: Wait until networkidle
const result = await scrapePage(page, url, 'networkidle', wait_after_load, timeout, check_selector);
pageContent = result.content;
pageStatusCode = result.status;
} catch (finalError) {
await page.close();
return res.status(500).json({ error: 'An error occurred while fetching the page.' });
}
}
const pageError = pageStatusCode !== 200 ? getError(pageStatusCode) : false;
if (!pageError) {
console.log(`✅ Scrape successful!`);
} else {
console.log(`🚨 Scrape failed with status code: ${pageStatusCode} ${pageError}`);
}
await page.close();
res.json({
content: pageContent,
pageStatusCode,
pageError
});
});
app.listen(port, () => {
initializeBrowser().then(() => {
console.log(`Server is running on port ${port}`);
});
});
process.on('SIGINT', () => {
shutdownBrowser().then(() => {
console.log('Browser closed');
process.exit(0);
});
});

73
apps/playwright-service-ts/helpers/get_error.ts Normal file
View File

@ -0,0 +1,73 @@
//impired by firecrawl repo @rafaelsideguide
export const getError = (statusCode: number | null): string | null => {
if (statusCode === null) {
return 'No response received';
}
const errorMessages: { [key: number]: string } = {
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
309: "Resume Incomplete",
310: "Too Many Redirects",
311: "Unavailable For Legal Reasons",
312: "Previously Used",
313: "I'm Used",
314: "Switch Proxy",
315: "Temporary Redirect",
316: "Resume Incomplete",
317: "Too Many Redirects",
400: "Bad Request",
401: "Unauthorized",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
599: "Network Connect Timeout Error"
};
if (statusCode < 300) {
return null;
}
return errorMessages[statusCode] || "Unknown Error";
};

28
apps/playwright-service-ts/package.json Normal file
View File

@ -0,0 +1,28 @@
{
"name": "playwright-scraper-api",
"version": "1.0.0",
"description": "scraper api with playwright",
"main": "api.ts",
"scripts": {
"start": "node dist/api.js",
"build": "tsc",
"dev": "ts-node api.ts"
},
"keywords": [],
"author": "Jeff Pereira",
"license": "ISC",
"dependencies": {
"body-parser": "^1.20.2",
"dotenv": "^16.4.5",
"express": "^4.19.2",
"playwright": "^1.45.0",
"random-useragent": "^0.5.0"
},
"devDependencies": {
"@types/express": "^4.17.21",
"@types/node": "^20.14.9",
"@types/random-useragent": "^0.3.3",
"ts-node": "^10.9.2",
"typescript": "^5.5.2"
}
}

110
apps/playwright-service-ts/tsconfig.json Normal file
View File

@ -0,0 +1,110 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
"module": "commonjs", /* Specify what module code is generated. */
"rootDir": "./", /* Specify the root folder within your source files. */
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
// "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
// "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
// "resolveJsonModule": true, /* Enable importing .json files. */
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "./dist", /* Specify an output folder for all emitted files. */
// "removeComments": true, /* Disable emitting comments. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
// "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
/* Type Checking */
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
},
"include": ["**/*"],
"exclude": ["node_modules", "**/*.spec.ts"]
}

6
apps/playwright-service/requirements.txt
View File

@ -1,4 +1,4 @@
hypercorn==0.16.0
fastapi==0.110.0
playwright==1.42.0
hypercorn==0.17.3
fastapi==0.111.0
playwright==1.44.0
uvicorn

2
apps/python-sdk/README.md
View File

@ -142,4 +142,4 @@ Contributions to the Firecrawl Python SDK are welcome! If you find any issues or
## License
The Firecrawl Python SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).
The Firecrawl Python SDK is open-source and released under the [AGPL License](https://www.gnu.org/licenses/agpl-3.0.en.html).

2
apps/python-sdk/pyproject.toml
View File

@ -13,7 +13,7 @@ dependencies = [
]
authors = [{name = "Mendable.ai",email = "nick@mendable.ai"}]
maintainers = [{name = "Mendable.ai",email = "nick@mendable.ai"}]
license = {text = "GNU General Public License v3 (GPLv3)"}
license = {text = "GNU Affero General Public License v3 (AGPLv3)"}
classifiers = [
"Development Status :: 5 - Production/Stable",

2
apps/test-suite/package.json
View File

@ -11,7 +11,7 @@
"author": "",
"license": "ISC",
"dependencies": {
"@anthropic-ai/sdk": "^0.20.8",
"@anthropic-ai/sdk": "^0.24.3",
"@dqbd/tiktoken": "^1.0.14",
"@supabase/supabase-js": "^2.43.1",
"dotenv": "^16.4.5",

3237
apps/test-suite/pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

75
docker-compose.yaml
View File

@ -1,5 +1,31 @@
name: firecrawl
version: '3.9'
x-common-service: &common-service
build: apps/api
networks:
- backend
environment:
- REDIS_URL=${REDIS_URL:-redis://redis:6379}
- PLAYWRIGHT_MICROSERVICE_URL=${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
- USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION}
- PORT=${PORT:-3002}
- NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}
- OPENAI_API_KEY=${OPENAI_API_KEY}
- SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
- SERPER_API_KEY=${SERPER_API_KEY}
- LLAMAPARSE_API_KEY=${LLAMAPARSE_API_KEY}
- LOGTAIL_KEY=${LOGTAIL_KEY}
- BULL_AUTH_KEY=${BULL_AUTH_KEY}
- TEST_API_KEY=${TEST_API_KEY}
- POSTHOG_API_KEY=${POSTHOG_API_KEY}
- POSTHOG_HOST=${POSTHOG_HOST}
- SUPABASE_ANON_TOKEN=${SUPABASE_ANON_TOKEN}
- SUPABASE_URL=${SUPABASE_URL}
- SUPABASE_SERVICE_TOKEN=${SUPABASE_SERVICE_TOKEN}
- SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}
- HOST=${HOST:-0.0.0.0}
services:
playwright-service:
build: apps/playwright-service
@ -13,64 +39,21 @@ services:
- backend
api:
build: apps/api
environment:
- REDIS_URL=${REDIS_URL:-redis://redis:6379}
- PLAYWRIGHT_MICROSERVICE_URL=${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
- USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION}
- PORT=${PORT:-3002}
- NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}
- OPENAI_API_KEY=${OPENAI_API_KEY}
- SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
- SERPER_API_KEY=${SERPER_API_KEY}
- LLAMAPARSE_API_KEY=${LLAMAPARSE_API_KEY}
- LOGTAIL_KEY=${LOGTAIL_KEY}
- BULL_AUTH_KEY=${BULL_AUTH_KEY}
- TEST_API_KEY=${TEST_API_KEY}
- POSTHOG_API_KEY=${POSTHOG_API_KEY}
- POSTHOG_HOST=${POSTHOG_HOST}
- SUPABASE_ANON_TOKEN=${SUPABASE_ANON_TOKEN}
- SUPABASE_URL=${SUPABASE_URL}
- SUPABASE_SERVICE_TOKEN=${SUPABASE_SERVICE_TOKEN}
- SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}
- HOST=${HOST:-0.0.0.0}
<<: *common-service
depends_on:
- redis
- playwright-service
ports:
- "3002:3002"
command: [ "pnpm", "run", "start:production" ]
networks:
- backend
worker:
build: apps/api
environment:
- REDIS_URL=${REDIS_URL:-redis://redis:6379}
- PLAYWRIGHT_MICROSERVICE_URL=${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
- USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION}
- PORT=${PORT:-3002}
- NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}
- OPENAI_API_KEY=${OPENAI_API_KEY}
- SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
- SERPER_API_KEY=${SERPER_API_KEY}
- LLAMAPARSE_API_KEY=${LLAMAPARSE_API_KEY}
- LOGTAIL_KEY=${LOGTAIL_KEY}
- BULL_AUTH_KEY=${BULL_AUTH_KEY}
- TEST_API_KEY=${TEST_API_KEY}
- POSTHOG_API_KEY=${POSTHOG_API_KEY}
- POSTHOG_HOST=${POSTHOG_HOST}
- SUPABASE_ANON_TOKEN=${SUPABASE_ANON_TOKEN}
- SUPABASE_URL=${SUPABASE_URL}
- SUPABASE_SERVICE_TOKEN=${SUPABASE_SERVICE_TOKEN}
- SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}
- HOST=${HOST:-0.0.0.0}
<<: *common-service
depends_on:
- redis
- playwright-service
- api
networks:
- backend
redis:
image: redis:alpine
networks: