diff --git a/package-lock.json b/package-lock.json index ba22bdf..35f1d1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "htmlparser2": "^9.0.0", "jose": "^5.1.0", "koa": "^2.16.0", + "koa-compress": "^5.1.1", "langdetect": "^0.2.1", "linkedom": "^0.18.4", "lru-cache": "^11.0.2", @@ -54,6 +55,7 @@ "@types/busboy": "^1.5.4", "@types/cors": "^2.8.17", "@types/koa": "^2.15.0", + "@types/koa-compress": "^4.0.6", "@types/node": "^20.14.13", "@types/set-cookie-parser": "^2.4.7", "@types/xmldom": "^0.1.34", @@ -2508,6 +2510,17 @@ "@types/koa": "*" } }, + "node_modules/@types/koa-compress": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/koa-compress/-/koa-compress-4.0.6.tgz", + "integrity": "sha512-jJzlsQTeIXRfQzY7y2Dl/syqOxvEaNJzFTJ8ZGaDFWvqgMmCXplfCGjPfHQgU7xYNKq/mCdN3YWus1YOldfIwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/koa": "*", + "@types/node": "*" + } + }, "node_modules/@types/lodash": { "version": "4.17.0", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", @@ -4197,6 +4210,18 @@ "node": ">= 12.0.0" } }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -8313,6 +8338,55 @@ "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==" }, + "node_modules/koa-compress": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/koa-compress/-/koa-compress-5.1.1.tgz", + "integrity": "sha512-UgMIN7ZoEP2DuoSQmD6CYvFSLt0NReGlc2qSY4bO4Oq0L56OiD9pDG41Kj/zFmVY/A3Wvmn4BqKcfq5H30LGIg==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "compressible": "^2.0.18", + "http-errors": "^1.8.1", + "koa-is-json": "^1.0.0" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/koa-compress/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/koa-compress/node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "license": "MIT", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/koa-compress/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/koa-convert": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz", @@ -8325,6 +8399,12 @@ "node": ">= 10" } }, + "node_modules/koa-is-json": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/koa-is-json/-/koa-is-json-1.0.0.tgz", + "integrity": "sha512-+97CtHAlWDx0ndt0J8y3P12EWLwTLMXIfMnYDev3wOTwH/RpBGMlfn4bDXlMEg1u73K6XRE9BbUp+5ZAYoRYWw==", + "license": "MIT" + }, "node_modules/koa/node_modules/http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", diff --git a/package.json b/package.json index fea444b..ae35f72 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "htmlparser2": "^9.0.0", "jose": "^5.1.0", "koa": "^2.16.0", + "koa-compress": "^5.1.1", "langdetect": "^0.2.1", "linkedom": "^0.18.4", "lru-cache": "^11.0.2", @@ -63,6 +64,7 @@ "@types/busboy": "^1.5.4", "@types/cors": "^2.8.17", "@types/koa": "^2.15.0", + "@types/koa-compress": "^4.0.6", "@types/node": "^20.14.13", "@types/set-cookie-parser": "^2.4.7", "@types/xmldom": "^0.1.34", diff --git a/src/stand-alone/crawl.ts b/src/stand-alone/crawl.ts index e2e5576..4a6e84c 100644 --- a/src/stand-alone/crawl.ts +++ b/src/stand-alone/crawl.ts @@ -18,6 +18,7 @@ import { ThreadedServiceRegistry } from '../services/threaded'; import { GlobalLogger } from '../services/logger'; import { AsyncLocalContext } from '../services/async-context'; import finalizer, { Finalizer } from '../services/finalizer'; +import koaCompress from 'koa-compress'; @singleton() export class CrawlStandAloneServer extends KoaServer { @@ -98,6 +99,23 @@ export class CrawlStandAloneServer extends KoaServer { } registerRoutes(): void { + this.koaApp.use(koaCompress({ + filter(type) { + if (type.startsWith('text/')) { + return true; + } + + if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { + return true; + } + + if (type.includes('application/x-ndjson')) { + return true; + } + + return false; + } + })); this.koaApp.use(this.makeAssetsServingController()); this.koaApp.use(this.registry.makeShimController()); } diff --git a/src/stand-alone/search.ts b/src/stand-alone/search.ts index b86840f..f817223 100644 --- a/src/stand-alone/search.ts +++ b/src/stand-alone/search.ts @@ -18,6 +18,7 @@ import { ThreadedServiceRegistry } from '../services/threaded'; import { GlobalLogger } from '../services/logger'; import { AsyncLocalContext } from '../services/async-context'; import finalizer, { Finalizer } from '../services/finalizer'; +import koaCompress from 'koa-compress'; @singleton() export class SearchStandAloneServer extends KoaServer { @@ -106,6 +107,23 @@ export class SearchStandAloneServer extends KoaServer { } registerRoutes(): void { + this.koaApp.use(koaCompress({ + filter(type) { + if (type.startsWith('text/')) { + return true; + } + + if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { + return true; + } + + if (type.includes('application/x-ndjson')) { + return true; + } + + return false; + } + })); this.koaApp.use(this.makeAssetsServingController()); this.koaApp.use(this.registry.makeShimController()); } diff --git a/src/stand-alone/serp.ts b/src/stand-alone/serp.ts index 557b4e6..0c039d7 100644 --- a/src/stand-alone/serp.ts +++ b/src/stand-alone/serp.ts @@ -18,6 +18,7 @@ import { GlobalLogger } from '../services/logger'; import { AsyncLocalContext } from '../services/async-context'; import finalizer, { Finalizer } from '../services/finalizer'; import { SerpHost } from '../api/serp'; +import koaCompress from 'koa-compress'; @singleton() export class SERPStandAloneServer extends KoaServer { @@ -106,6 +107,23 @@ export class SERPStandAloneServer extends KoaServer { } registerRoutes(): void { + this.koaApp.use(koaCompress({ + filter(type) { + if (type.startsWith('text/')) { + return true; + } + + if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { + return true; + } + + if (type.includes('application/x-ndjson')) { + return true; + } + + return false; + } + })); this.koaApp.use(this.makeAssetsServingController()); this.koaApp.use(this.registry.makeShimController()); }