AI/reader
1
0
Fork 0
mirror of https://git-proxy.hk.martin98.com/https://github.com/jina-ai/reader synced 2025-04-10 13:10:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

saas: use compression

Browse Source
This commit is contained in:
Yanlong Wang 2025-04-02 19:16:42 +08:00
parent 1bf0568489
commit f9223d78eb
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
5 changed files with 136 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
package-lock.json generated
View File

@ -26,6 +26,7 @@
"htmlparser2": "^9.0.0",
"jose": "^5.1.0",
"koa": "^2.16.0",
"koa-compress": "^5.1.1",
"langdetect": "^0.2.1",
"linkedom": "^0.18.4",
"lru-cache": "^11.0.2",
@ -54,6 +55,7 @@
"@types/busboy": "^1.5.4",
"@types/cors": "^2.8.17",
"@types/koa": "^2.15.0",
"@types/koa-compress": "^4.0.6",
"@types/node": "^20.14.13",
"@types/set-cookie-parser": "^2.4.7",
"@types/xmldom": "^0.1.34",
@ -2508,6 +2510,17 @@
"@types/koa": "*"
}
},
"node_modules/@types/koa-compress": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/@types/koa-compress/-/koa-compress-4.0.6.tgz",
"integrity": "sha512-jJzlsQTeIXRfQzY7y2Dl/syqOxvEaNJzFTJ8ZGaDFWvqgMmCXplfCGjPfHQgU7xYNKq/mCdN3YWus1YOldfIwg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/koa": "*",
"@types/node": "*"
}
},
"node_modules/@types/lodash": {
"version": "4.17.0",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz",
@ -4197,6 +4210,18 @@
"node": ">= 12.0.0"
}
},
"node_modules/compressible": {
"version": "2.0.18",
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
"integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
"license": "MIT",
"dependencies": {
"mime-db": ">= 1.43.0 < 2"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@ -8313,6 +8338,55 @@
"resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz",
"integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw=="
},
"node_modules/koa-compress": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/koa-compress/-/koa-compress-5.1.1.tgz",
"integrity": "sha512-UgMIN7ZoEP2DuoSQmD6CYvFSLt0NReGlc2qSY4bO4Oq0L56OiD9pDG41Kj/zFmVY/A3Wvmn4BqKcfq5H30LGIg==",
"license": "MIT",
"dependencies": {
"bytes": "^3.1.2",
"compressible": "^2.0.18",
"http-errors": "^1.8.1",
"koa-is-json": "^1.0.0"
},
"engines": {
"node": ">= 12"
}
},
"node_modules/koa-compress/node_modules/depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
"integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/koa-compress/node_modules/http-errors": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
"integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
"license": "MIT",
"dependencies": {
"depd": "~1.1.2",
"inherits": "2.0.4",
"setprototypeof": "1.2.0",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.1"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/koa-compress/node_modules/statuses": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/koa-convert": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz",
@ -8325,6 +8399,12 @@
"node": ">= 10"
}
},
"node_modules/koa-is-json": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/koa-is-json/-/koa-is-json-1.0.0.tgz",
"integrity": "sha512-+97CtHAlWDx0ndt0J8y3P12EWLwTLMXIfMnYDev3wOTwH/RpBGMlfn4bDXlMEg1u73K6XRE9BbUp+5ZAYoRYWw==",
"license": "MIT"
},
"node_modules/koa/node_modules/http-errors": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",

2
package.json
View File

@ -35,6 +35,7 @@
"htmlparser2": "^9.0.0",
"jose": "^5.1.0",
"koa": "^2.16.0",
"koa-compress": "^5.1.1",
"langdetect": "^0.2.1",
"linkedom": "^0.18.4",
"lru-cache": "^11.0.2",
@ -63,6 +64,7 @@
"@types/busboy": "^1.5.4",
"@types/cors": "^2.8.17",
"@types/koa": "^2.15.0",
"@types/koa-compress": "^4.0.6",
"@types/node": "^20.14.13",
"@types/set-cookie-parser": "^2.4.7",
"@types/xmldom": "^0.1.34",

18
src/stand-alone/crawl.ts
View File

@ -18,6 +18,7 @@ import { ThreadedServiceRegistry } from '../services/threaded';
import { GlobalLogger } from '../services/logger';
import { AsyncLocalContext } from '../services/async-context';
import finalizer, { Finalizer } from '../services/finalizer';
import koaCompress from 'koa-compress';
@singleton()
export class CrawlStandAloneServer extends KoaServer {
@ -98,6 +99,23 @@ export class CrawlStandAloneServer extends KoaServer {
}
registerRoutes(): void {
this.koaApp.use(koaCompress({
filter(type) {
if (type.startsWith('text/')) {
return true;
}
if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
return true;
}
if (type.includes('application/x-ndjson')) {
return true;
}
return false;
}
}));
this.koaApp.use(this.makeAssetsServingController());
this.koaApp.use(this.registry.makeShimController());
}

18
src/stand-alone/search.ts
View File

@ -18,6 +18,7 @@ import { ThreadedServiceRegistry } from '../services/threaded';
import { GlobalLogger } from '../services/logger';
import { AsyncLocalContext } from '../services/async-context';
import finalizer, { Finalizer } from '../services/finalizer';
import koaCompress from 'koa-compress';
@singleton()
export class SearchStandAloneServer extends KoaServer {
@ -106,6 +107,23 @@ export class SearchStandAloneServer extends KoaServer {
}
registerRoutes(): void {
this.koaApp.use(koaCompress({
filter(type) {
if (type.startsWith('text/')) {
return true;
}
if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
return true;
}
if (type.includes('application/x-ndjson')) {
return true;
}
return false;
}
}));
this.koaApp.use(this.makeAssetsServingController());
this.koaApp.use(this.registry.makeShimController());
}

18
src/stand-alone/serp.ts
View File

@ -18,6 +18,7 @@ import { GlobalLogger } from '../services/logger';
import { AsyncLocalContext } from '../services/async-context';
import finalizer, { Finalizer } from '../services/finalizer';
import { SerpHost } from '../api/serp';
import koaCompress from 'koa-compress';
@singleton()
export class SERPStandAloneServer extends KoaServer {
@ -106,6 +107,23 @@ export class SERPStandAloneServer extends KoaServer {
}
registerRoutes(): void {
this.koaApp.use(koaCompress({
filter(type) {
if (type.startsWith('text/')) {
return true;
}
if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
return true;
}
if (type.includes('application/x-ndjson')) {
return true;
}
return false;
}
}));
this.koaApp.use(this.makeAssetsServingController());
this.koaApp.use(this.registry.makeShimController());
}