import request from "supertest"; import { configDotenv } from "dotenv"; import { ScrapeRequestInput } from "../../controllers/v1/types"; configDotenv(); const TEST_URL = "http://127.0.0.1:3002"; async function scrape(body: ScrapeRequestInput) { return await request(TEST_URL) .post("/v1/scrape") .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Content-Type", "application/json") .send(body); } function expectScrapeToSucceed(response: Awaited>) { expect(response.statusCode).toBe(200); expect(response.body.success).toBe(true); expect(typeof response.body.data).toBe("object"); } describe("Scrape tests", () => { it("mocking works properly", async () => { // depends on falsified mock mocking-works-properly // this test will fail if mock is bypassed with real data -- firecrawl.dev will never have // that as its actual markdown output const response = await scrape({ url: "http://firecrawl.dev", useMock: "mocking-works-properly", }); expectScrapeToSucceed(response); expect(response.body.data.markdown).toBe( "this is fake data coming from the mocking system!", ); }, 10000); describe("Ad blocking (f-e dependant)", () => { it.concurrent("blocks ads by default", async () => { const response = await scrape({ url: "https://canyoublockit.com/testing/", }); expectScrapeToSucceed(response); expect(response.body.data.markdown).not.toContain(".g.doubleclick.net/"); }, 10000); it.concurrent("doesn't block ads if explicitly disabled", async () => { const response = await scrape({ url: "https://canyoublockit.com/testing/", blockAds: false, }); expectScrapeToSucceed(response); expect(response.body.data.markdown).toContain(".g.doubleclick.net/"); }, 10000); }); describe("Location API (f-e dependant)", () => { it.concurrent("works without specifying an explicit location", async () => { const response = await scrape({ url: "https://iplocation.com", }); expectScrapeToSucceed(response); }, 10000); it.concurrent("works with country US", async () => { const response = await scrape({ url: "https://iplocation.com", location: { country: "US" }, }); expectScrapeToSucceed(response); expect(response.body.data.markdown).toContain("| Country | United States |"); }, 10000); }); describe("JSON scrape support", () => { it.concurrent("returns parseable JSON", async () => { const response = await scrape({ url: "https://jsonplaceholder.typicode.com/todos/1", formats: ["rawHtml"], }); expectScrapeToSucceed(response); const obj = JSON.parse(response.body.data.rawHtml); expect(obj.id).toBe(1); }, 25000); // TODO: mock and shorten }); describe("Screenshot", () => { it.concurrent("screenshot format works", async () => { const response = await scrape({ url: "http://firecrawl.dev", formats: ["screenshot"] }); expectScrapeToSucceed(response); expect(typeof response.body.data.screenshot).toBe("string"); }, 15000); it.concurrent("screenshot@fullPage format works", async () => { const response = await scrape({ url: "http://firecrawl.dev", formats: ["screenshot@fullPage"] }); expectScrapeToSucceed(response); expect(typeof response.body.data.screenshot).toBe("string"); }, 15000); }); describe("Proxy API (f-e dependant)", () => { it.concurrent("undefined works", async () => { await scrape({ url: "http://firecrawl.dev", }); }, 15000); it.concurrent("basic works", async () => { await scrape({ url: "http://firecrawl.dev", proxy: "basic", }); }, 15000); it.concurrent("stealth works", async () => { await scrape({ url: "http://firecrawl.dev", proxy: "stealth", }); }, 15000); }); });