Revert "feat: use cacheable lookup everywhere (#1559)"

This reverts commit b8703b2a720765b92f5c4cab94cc90ea624198a8.

.github/workflows/test-server.yml

@@ -35,7 +35,6 @@ env:
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
   VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
   USE_GO_MARKDOWN_PARSER: true
-  SENTRY_ENVIRONMENT: dev
 
 jobs:
   test:
@@ -54,7 +53,6 @@ jobs:
           oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
           oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
           tags: tag:ci
-          use-cache: 'true'
       - name: Install pnpm
         uses: pnpm/action-setup@v4
         with:

apps/api/src/index.ts

@@ -18,6 +18,7 @@ import { logger } from "./lib/logger";
 import { adminRouter } from "./routes/admin";
 import http from "node:http";
 import https from "node:https";
+import CacheableLookup from "cacheable-lookup";
 import { v1Router } from "./routes/v1";
 import expressWs from "express-ws";
 import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";
@@ -25,7 +26,6 @@ import { ZodError } from "zod";
 import { v4 as uuidv4 } from "uuid";
 import { RateLimiterMode } from "./types";
 import { attachWsProxy } from "./services/agentLivecastWS";
-import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
 
 const { createBullBoard } = require("@bull-board/api");
 const { BullAdapter } = require("@bull-board/api/bullAdapter");
@@ -34,9 +34,11 @@ const { ExpressAdapter } = require("@bull-board/express");
 const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;
 logger.info(`Number of CPUs: ${numCPUs} available`);
 
+const cacheable = new CacheableLookup();
+
 // Install cacheable lookup for all other requests
-cacheableLookup.install(http.globalAgent);
+cacheable.install(http.globalAgent);
-cacheableLookup.install(https.globalAgent);
+cacheable.install(https.globalAgent);
 
 // Initialize Express with WebSocket support
 const expressApp = express();

apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts

@@ -2,7 +2,6 @@ import type { Socket } from "net";
 import type { TLSSocket } from "tls";
 import * as undici from "undici";
 import { Address6 } from "ip-address";
-import { cacheableLookup } from "../../lib/cacheableLookup";
 
 export class InsecureConnectionError extends Error {
   constructor() {
@@ -47,7 +46,7 @@ export function makeSecureDispatcher(
   const agentOpts: undici.Agent.Options = {
     connect: {
       rejectUnauthorized: false, // bypass SSL failures -- this is fine
-      lookup: cacheableLookup.lookup,
+      // lookup: secureLookup,
     },
     maxRedirections: 5000,
     ...options,

apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts

@@ -1,4 +0,0 @@
-import CacheableLookup from 'cacheable-lookup';
-import dns from 'dns';
-
-export const cacheableLookup = (process.env.SENTRY_ENVIRONMENT === "dev" ? { lookup: dns.lookup, install: () => {} } : new CacheableLookup({ lookup: false }));

apps/api/src/scraper/scrapeURL/lib/fetch.ts

@@ -5,7 +5,6 @@ import { MockState, saveMock } from "./mock";
 import { TimeoutSignal } from "../../../controllers/v1/types";
 import { fireEngineURL } from "../engines/fire-engine/scrape";
 import { fetch, RequestInit, Response, FormData, Agent } from "undici";
-import { cacheableLookup } from "./cacheableLookup";
 
 export type RobustFetchParams<Schema extends z.Schema<any>> = {
   url: string;
@@ -83,9 +82,6 @@ export async function robustFetch<
         dispatcher: new Agent({
           headersTimeout: 0,
           bodyTimeout: 0,
-          connect: {
-            lookup: cacheableLookup.lookup,
-          },
         }),
         ...(body instanceof FormData
           ? {

apps/api/src/services/queue-worker.ts

@@ -82,9 +82,6 @@ import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f
 import { CostTracking } from "../lib/extract/extraction-service";
 import { getACUCTeam } from "../controllers/auth";
 import Express from "express";
-import http from "http";
-import https from "https";
-import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";
 
 configDotenv();
 
@@ -112,10 +109,6 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;
 
 const runningJobs: Set<string> = new Set();
 
-// Install cacheable lookup for all other requests
-cacheableLookup.install(http.globalAgent);
-cacheableLookup.install(https.globalAgent);
-
 async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
   const logger = _logger.child({
     module: "queue-worker",