From 9fd735f3a1fbda29ce34117f70bbe68b7537ffb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 9 Apr 2025 15:45:07 +0200 Subject: [PATCH] feat(api/test/snips): disable flaky tests --- apps/api/src/__tests__/snips/billing.test.ts | 315 ++++++++++--------- apps/api/src/__tests__/snips/scrape.test.ts | 21 +- 2 files changed, 171 insertions(+), 165 deletions(-) diff --git a/apps/api/src/__tests__/snips/billing.test.ts b/apps/api/src/__tests__/snips/billing.test.ts index 8afa5a44..9d183e6d 100644 --- a/apps/api/src/__tests__/snips/billing.test.ts +++ b/apps/api/src/__tests__/snips/billing.test.ts @@ -1,192 +1,197 @@ -import { batchScrape, crawl, creditUsage, extract, map, scrape, search, tokenUsage } from "./lib"; +// import { batchScrape, crawl, creditUsage, extract, map, scrape, search, tokenUsage } from "./lib"; -const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms)); -const sleepForBatchBilling = () => sleep(20000); +// const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms)); +// const sleepForBatchBilling = () => sleep(20000); -beforeAll(async () => { - // Wait for previous test runs to stop billing processing - if (!process.env.TEST_SUITE_SELF_HOSTED) { - await sleep(40000); - } -}, 50000); +// beforeAll(async () => { +// // Wait for previous test runs to stop billing processing +// if (!process.env.TEST_SUITE_SELF_HOSTED) { +// await sleep(40000); +// } +// }, 50000); -describe("Billing tests", () => { - if (process.env.TEST_SUITE_SELF_HOSTED) { - it("dummy", () => { - expect(true).toBe(true); - }); - } else { - it("bills scrape correctly", async () => { - const rc1 = (await creditUsage()).remaining_credits; +// describe("Billing tests", () => { +// if (process.env.TEST_SUITE_SELF_HOSTED) { +// it("dummy", () => { +// expect(true).toBe(true); +// }); +// } else { +// it("bills scrape correctly", async () => { +// const rc1 = (await creditUsage()).remaining_credits; - // Run all scrape operations in parallel with Promise.all - await Promise.all([ - // scrape 1: regular fc.dev scrape (1 credit) - scrape({ - url: "https://firecrawl.dev" - }), +// // Run all scrape operations in parallel with Promise.all +// await Promise.all([ +// // scrape 1: regular fc.dev scrape (1 credit) +// scrape({ +// url: "https://firecrawl.dev" +// }), - // scrape 1.1: regular fc.dev scrape (1 credit) - scrape({ - url: "https://firecrawl.dev" - }), +// // scrape 1.1: regular fc.dev scrape (1 credit) +// scrape({ +// url: "https://firecrawl.dev" +// }), - // scrape 2: fc.dev with json (5 credits) - scrape({ - url: "https://firecrawl.dev", - formats: ["json"], - jsonOptions: { - schema: { - type: "object", - properties: { - is_open_source: { type: "boolean" }, - }, - required: ["is_open_source"], - }, - }, - }) - ]); +// // scrape 2: fc.dev with json (5 credits) +// scrape({ +// url: "https://firecrawl.dev", +// formats: ["json"], +// jsonOptions: { +// schema: { +// type: "object", +// properties: { +// is_open_source: { type: "boolean" }, +// }, +// required: ["is_open_source"], +// }, +// }, +// }) +// ]); - // sum: 7 credits +// // sum: 7 credits - await sleepForBatchBilling(); +// await sleepForBatchBilling(); - const rc2 = (await creditUsage()).remaining_credits; +// const rc2 = (await creditUsage()).remaining_credits; - expect(rc1 - rc2).toBe(7); - }, 120000); +// expect(rc1 - rc2).toBe(7); +// }, 120000); - it("bills batch scrape correctly", async () => { - const rc1 = (await creditUsage()).remaining_credits; +// it("bills batch scrape correctly", async () => { +// const rc1 = (await creditUsage()).remaining_credits; - // Run both scrape operations in parallel with Promise.all - const [scrape1, scrape2] = await Promise.all([ - // scrape 1: regular batch scrape with failing domain (2 credits) - batchScrape({ - urls: [ - "https://firecrawl.dev", - "https://mendable.ai", - "https://thisdomaindoesnotexistandwillfail.fcr", - ], - }), +// // Run both scrape operations in parallel with Promise.all +// const [scrape1, scrape2] = await Promise.all([ +// // scrape 1: regular batch scrape with failing domain (2 credits) +// batchScrape({ +// urls: [ +// "https://firecrawl.dev", +// "https://mendable.ai", +// "https://thisdomaindoesnotexistandwillfail.fcr", +// ], +// }), - // scrape 2: batch scrape with json (10 credits) - batchScrape({ - urls: [ - "https://firecrawl.dev", - "https://mendable.ai", - "https://thisdomaindoesnotexistandwillfail.fcr", - ], - formats: ["json"], - jsonOptions: { - schema: { - type: "object", - properties: { - four_word_summary: { type: "string" }, - }, - required: ["four_word_summary"], - }, - }, - }) - ]); +// // scrape 2: batch scrape with json (10 credits) +// batchScrape({ +// urls: [ +// "https://firecrawl.dev", +// "https://mendable.ai", +// "https://thisdomaindoesnotexistandwillfail.fcr", +// ], +// formats: ["json"], +// jsonOptions: { +// schema: { +// type: "object", +// properties: { +// four_word_summary: { type: "string" }, +// }, +// required: ["four_word_summary"], +// }, +// }, +// }) +// ]); - // sum: 12 credits +// // sum: 12 credits - await sleepForBatchBilling(); +// await sleepForBatchBilling(); - const rc2 = (await creditUsage()).remaining_credits; +// const rc2 = (await creditUsage()).remaining_credits; - expect(rc1 - rc2).toBe(12); - }, 600000); +// expect(rc1 - rc2).toBe(12); +// }, 600000); - it("bills crawl correctly", async () => { - const rc1 = (await creditUsage()).remaining_credits; +// it("bills crawl correctly", async () => { +// const rc1 = (await creditUsage()).remaining_credits; - // Run both crawl operations in parallel with Promise.all - const [crawl1, crawl2] = await Promise.all([ - // crawl 1: regular fc.dev crawl (x credits) - crawl({ - url: "https://firecrawl.dev", - }), +// // Run both crawl operations in parallel with Promise.all +// const [crawl1, crawl2] = await Promise.all([ +// // crawl 1: regular fc.dev crawl (x credits) +// crawl({ +// url: "https://firecrawl.dev", +// }), - // crawl 2: fc.dev crawl with json (5y credits) - crawl({ - url: "https://firecrawl.dev", - scrapeOptions: { - formats: ["json"], - jsonOptions: { - schema: { - type: "object", - properties: { - four_word_summary: { type: "string" }, - }, - required: ["four_word_summary"], - }, - }, - } - }) - ]); +// // crawl 2: fc.dev crawl with json (5y credits) +// crawl({ +// url: "https://firecrawl.dev", +// scrapeOptions: { +// formats: ["json"], +// jsonOptions: { +// schema: { +// type: "object", +// properties: { +// four_word_summary: { type: "string" }, +// }, +// required: ["four_word_summary"], +// }, +// }, +// } +// }) +// ]); - expect(crawl1.success).toBe(true); - expect(crawl2.success).toBe(true); +// expect(crawl1.success).toBe(true); +// expect(crawl2.success).toBe(true); - // sum: x+5y credits +// // sum: x+5y credits - await sleepForBatchBilling(); +// await sleepForBatchBilling(); - const rc2 = (await creditUsage()).remaining_credits; +// const rc2 = (await creditUsage()).remaining_credits; - if (crawl1.success && crawl2.success) { - expect(rc1 - rc2).toBe(crawl1.completed + crawl2.completed * 5); - } - }, 600000); +// if (crawl1.success && crawl2.success) { +// expect(rc1 - rc2).toBe(crawl1.completed + crawl2.completed * 5); +// } +// }, 600000); - it("bills map correctly", async () => { - const rc1 = (await creditUsage()).remaining_credits; - await map({ url: "https://firecrawl.dev" }); - await sleepForBatchBilling(); - const rc2 = (await creditUsage()).remaining_credits; - expect(rc1 - rc2).toBe(1); - }, 60000); +// it("bills map correctly", async () => { +// const rc1 = (await creditUsage()).remaining_credits; +// await map({ url: "https://firecrawl.dev" }); +// await sleepForBatchBilling(); +// const rc2 = (await creditUsage()).remaining_credits; +// expect(rc1 - rc2).toBe(1); +// }, 60000); - it("bills search correctly", async () => { - const rc1 = (await creditUsage()).remaining_credits; +// it("bills search correctly", async () => { +// const rc1 = (await creditUsage()).remaining_credits; - const results = await search({ - query: "firecrawl" - }); +// const results = await search({ +// query: "firecrawl" +// }); - await sleepForBatchBilling(); +// await sleepForBatchBilling(); - const rc2 = (await creditUsage()).remaining_credits; +// const rc2 = (await creditUsage()).remaining_credits; - expect(rc1 - rc2).toBe(results.length); - }, 60000); +// expect(rc1 - rc2).toBe(results.length); +// }, 60000); - it("bills extract correctly", async () => { - const rc1 = (await tokenUsage()).remaining_tokens; +// it("bills extract correctly", async () => { +// const rc1 = (await tokenUsage()).remaining_tokens; - await extract({ - urls: ["https://firecrawl.dev"], - schema: { - "type": "object", - "properties": { - "is_open_source": { - "type": "boolean" - } - }, - "required": [ - "is_open_source" - ] - }, - origin: "api-sdk", - }); +// await extract({ +// urls: ["https://firecrawl.dev"], +// schema: { +// "type": "object", +// "properties": { +// "is_open_source": { +// "type": "boolean" +// } +// }, +// "required": [ +// "is_open_source" +// ] +// }, +// origin: "api-sdk", +// }); - await sleepForBatchBilling(); +// await sleepForBatchBilling(); - const rc2 = (await tokenUsage()).remaining_tokens; +// const rc2 = (await tokenUsage()).remaining_tokens; - expect(rc1 - rc2).toBe(305); - }, 300000); - } +// expect(rc1 - rc2).toBe(305); +// }, 300000); +// } +// }); + +// temporarily disabled +it("is mocked", () => { + expect(true).toBe(true); }); \ No newline at end of file diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts index bb053271..96337171 100644 --- a/apps/api/src/__tests__/snips/scrape.test.ts +++ b/apps/api/src/__tests__/snips/scrape.test.ts @@ -152,20 +152,21 @@ describe("Scrape tests", () => { await scrape({ url: "http://firecrawl.dev", proxy: "stealth", - timeout: 60000, + timeout: 120000, }); - }, 70000); + }, 130000); }); - describe("PDF (f-e dependant)", () => { - it.concurrent("works for PDFs behind anti-bot", async () => { - const response = await scrape({ - url: "https://www.researchgate.net/profile/Amir-Leshem/publication/220732050_Robust_adaptive_beamforming_based_on_jointly_estimating_covariance_matrix_and_steering_vector/links/0c96052d2fd8f0a84b000000/Robust-adaptive-beamforming-based-on-jointly-estimating-covariance-matrix-and-steering-vector.pdf" - }); + // Temporarily disabled, too flaky + // describe("PDF (f-e dependant)", () => { + // it.concurrent("works for PDFs behind anti-bot", async () => { + // const response = await scrape({ + // url: "https://www.researchgate.net/profile/Amir-Leshem/publication/220732050_Robust_adaptive_beamforming_based_on_jointly_estimating_covariance_matrix_and_steering_vector/links/0c96052d2fd8f0a84b000000/Robust-adaptive-beamforming-based-on-jointly-estimating-covariance-matrix-and-steering-vector.pdf" + // }); - expect(response.markdown).toContain("Robust adaptive beamforming based on jointly estimating covariance matrix"); - }, 60000); - }); + // expect(response.markdown).toContain("Robust adaptive beamforming based on jointly estimating covariance matrix"); + // }, 60000); + // }); } if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {