From 04942bb9dea14a43ac26bc434ef8efa3eadbabbd Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Jul 2024 18:31:43 -0400 Subject: [PATCH 1/3] Nick: --- apps/api/fly.toml | 4 ++-- apps/api/package.json | 1 + apps/api/pnpm-lock.yaml | 17 +++++++++++++---- apps/api/src/controllers/auth.ts | 21 +++++++++++++++++++-- apps/api/src/index.ts | 11 +++++++++++ 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/apps/api/fly.toml b/apps/api/fly.toml index e248e59b..94108e5f 100644 --- a/apps/api/fly.toml +++ b/apps/api/fly.toml @@ -24,8 +24,8 @@ kill_timeout = '30s' [http_service.concurrency] type = "requests" - hard_limit = 100 - soft_limit = 50 + hard_limit = 200 + soft_limit = 75 [[http_service.checks]] grace_period = "20s" diff --git a/apps/api/package.json b/apps/api/package.json index da1b2b33..153ac841 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -63,6 +63,7 @@ "axios": "^1.3.4", "bottleneck": "^2.19.5", "bull": "^4.15.0", + "cacheable-lookup": "^6.1.0", "cheerio": "^1.0.0-rc.12", "cohere": "^1.1.1", "cors": "^2.8.5", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 02d8363b..9c8dbbc9 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: bull: specifier: ^4.15.0 version: 4.15.0 + cacheable-lookup: + specifier: ^6.1.0 + version: 6.1.0 cheerio: specifier: ^1.0.0-rc.12 version: 1.0.0-rc.12 @@ -1937,6 +1940,10 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} + cacheable-lookup@6.1.0: + resolution: {integrity: sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==} + engines: {node: '>=10.6.0'} + call-bind@1.0.7: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} engines: {node: '>= 0.4'} @@ -4369,8 +4376,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.5.3: - resolution: {integrity: sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==} + typescript@5.5.4: + resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} engines: {node: '>=14.17'} hasBin: true @@ -6917,6 +6924,8 @@ snapshots: bytes@3.1.2: {} + cacheable-lookup@6.1.0: {} + call-bind@1.0.7: dependencies: es-define-property: 1.0.0 @@ -8927,7 +8936,7 @@ snapshots: csv-parse: 5.5.6 gpt3-tokenizer: 1.1.5 openai: 3.3.0 - typescript: 5.5.3 + typescript: 5.5.4 uuid: 9.0.1 zod: 3.23.8 transitivePeerDependencies: @@ -9519,7 +9528,7 @@ snapshots: typescript@5.4.5: {} - typescript@5.5.3: {} + typescript@5.5.4: {} typesense@1.8.2(@babel/runtime@7.24.6): dependencies: diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index aab3d188..7154c694 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -88,7 +88,15 @@ export async function supaAuthenticateUser( // console.log('Key and Price ID:', data); } - if (error || !data || data.length === 0) { + if(error) { + return { + success: false, + error: "Unauthorized: Invalid token or server error.", + status: 500, + }; + } + + if (!data || data.length === 0) { return { success: false, error: "Unauthorized: Invalid token", @@ -178,7 +186,16 @@ export async function supaAuthenticateUser( .select("*") .eq("key", normalizedApi); - if (error || !data || data.length === 0) { + if (error) { + Logger.warn(`Error fetching api key: ${error.message}`); + return { + success: false, + error: "Unauthorized: Invalid token or server error.", + status: 500, + }; + } + + if (!data || data.length === 0) { return { success: false, error: "Unauthorized: Invalid token", diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 3fba86f8..ebe6ef38 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -10,6 +10,9 @@ import os from "os"; import { Logger } from "./lib/logger"; import { adminRouter } from "./routes/admin"; import { ScrapeEvents } from "./lib/scrape-events"; +import http from 'node:http'; +import https from 'node:https'; +import CacheableLookup from 'cacheable-lookup'; const { createBullBoard } = require("@bull-board/api"); const { BullAdapter } = require("@bull-board/api/bullAdapter"); @@ -18,6 +21,14 @@ const { ExpressAdapter } = require("@bull-board/express"); const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length; Logger.info(`Number of CPUs: ${numCPUs} available`); +const cacheable = new CacheableLookup({ + // this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme + lookup:false +}); + +cacheable.install(http.globalAgent); +cacheable.install(https.globalAgent) + if (cluster.isMaster) { Logger.info(`Master ${process.pid} is running`); From 968a2dc75330968953569dd1856e4da27953f8e7 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Jul 2024 18:37:09 -0400 Subject: [PATCH 2/3] Nick: --- apps/api/package.json | 1 + apps/api/pnpm-lock.yaml | 3 +++ .../src/scraper/WebScraper/__tests__/dns.test.ts | 15 +++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 apps/api/src/scraper/WebScraper/__tests__/dns.test.ts diff --git a/apps/api/package.json b/apps/api/package.json index 153ac841..15e97377 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -26,6 +26,7 @@ "license": "ISC", "devDependencies": { "@flydotio/dockerfile": "^0.4.10", + "@jest/globals": "^29.7.0", "@tsconfig/recommended": "^1.0.3", "@types/body-parser": "^1.19.2", "@types/bull": "^4.10.0", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 9c8dbbc9..ec83e18b 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -192,6 +192,9 @@ importers: '@flydotio/dockerfile': specifier: ^0.4.10 version: 0.4.11 + '@jest/globals': + specifier: ^29.7.0 + version: 29.7.0 '@tsconfig/recommended': specifier: ^1.0.3 version: 1.0.6 diff --git a/apps/api/src/scraper/WebScraper/__tests__/dns.test.ts b/apps/api/src/scraper/WebScraper/__tests__/dns.test.ts new file mode 100644 index 00000000..968ed121 --- /dev/null +++ b/apps/api/src/scraper/WebScraper/__tests__/dns.test.ts @@ -0,0 +1,15 @@ +import CacheableLookup from 'cacheable-lookup'; +import https from 'node:https'; +import axios from "axios"; + +describe("DNS", () => { + it("cached dns", async () => { + const cachedDns = new CacheableLookup(); + cachedDns.install(https.globalAgent); + jest.spyOn(cachedDns, "lookupAsync"); + + const res = await axios.get("https://example.com"); + expect(res.status).toBe(200); + expect(cachedDns.lookupAsync).toHaveBeenCalled(); + }); +}); From e99c2568f4401634d43d6c10aa495d68370c0ed1 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Jul 2024 18:44:18 -0400 Subject: [PATCH 3/3] Update auth.ts --- apps/api/src/controllers/auth.ts | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 7154c694..035d847d 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -88,15 +88,10 @@ export async function supaAuthenticateUser( // console.log('Key and Price ID:', data); } - if(error) { - return { - success: false, - error: "Unauthorized: Invalid token or server error.", - status: 500, - }; - } + - if (!data || data.length === 0) { + if (error || !data || data.length === 0) { + Logger.warn(`Error fetching api key: ${error.message} or data is empty`); return { success: false, error: "Unauthorized: Invalid token", @@ -186,16 +181,10 @@ export async function supaAuthenticateUser( .select("*") .eq("key", normalizedApi); - if (error) { - Logger.warn(`Error fetching api key: ${error.message}`); - return { - success: false, - error: "Unauthorized: Invalid token or server error.", - status: 500, - }; - } + - if (!data || data.length === 0) { + if (error || !data || data.length === 0) { + Logger.warn(`Error fetching api key: ${error.message} or data is empty`); return { success: false, error: "Unauthorized: Invalid token",