GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-18 01:05:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

curl-impersonate: add zstd compression options

Browse Source
This commit is contained in:
yanlong.wang 2025-02-19 14:22:19 +08:00
parent 2991e300d8
commit b6846ab4b6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
5 changed files with 135 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/functions/Dockerfile
View File

@ -8,7 +8,7 @@ RUN apt-get update \
&& wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
&& sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
&& apt-get update \
&& apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
&& apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*

120
backend/functions/package-lock.json generated
View File

@ -40,6 +40,7 @@
"puppeteer-page-proxy": "^1.3.0",
"robots-parser": "^3.0.1",
"set-cookie-parser": "^2.6.0",
"simple-zstd": "^1.4.2",
"stripe": "^11.11.0",
"tiktoken": "^1.0.16",
"tld-extract": "^2.1.0",
@ -3701,9 +3702,7 @@
"node_modules/buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
"dev": true,
"peer": true
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
},
"node_modules/busboy": {
"version": "1.6.0",
@ -4761,6 +4760,12 @@
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/duplex-maker": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/duplex-maker/-/duplex-maker-1.0.0.tgz",
"integrity": "sha512-KoHuzggxg7f+vvjqOHfXxaQYI1POzBm+ah0eec7YDssZmbt6QFBI8d1nl5GQwAgR2f+VQCPvyvZtmWWqWuFtlA==",
"license": "MIT"
},
"node_modules/duplexify": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz",
@ -7288,6 +7293,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/is-zst": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-zst/-/is-zst-1.0.0.tgz",
"integrity": "sha512-ZA5lvshKAl8z30dX7saXLpVhpsq3d2EHK9uf7qtUjnOtdw4XBpAoWb2RvZ5kyoaebdoidnGI0g2hn9Z7ObPbww==",
"license": "MIT"
},
"node_modules/isarray": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
@ -10043,6 +10054,75 @@
"path2d": "^0.2.0"
}
},
"node_modules/peek-stream": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
"integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==",
"license": "MIT",
"dependencies": {
"buffer-from": "^1.0.0",
"duplexify": "^3.5.0",
"through2": "^2.0.3"
}
},
"node_modules/peek-stream/node_modules/duplexify": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
"integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
"license": "MIT",
"dependencies": {
"end-of-stream": "^1.0.0",
"inherits": "^2.0.1",
"readable-stream": "^2.0.0",
"stream-shift": "^1.0.0"
}
},
"node_modules/peek-stream/node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
"license": "MIT"
},
"node_modules/peek-stream/node_modules/readable-stream": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
"license": "MIT",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/peek-stream/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"license": "MIT"
},
"node_modules/peek-stream/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"license": "MIT",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/peek-stream/node_modules/through2": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",
"integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==",
"license": "MIT",
"dependencies": {
"readable-stream": "~2.3.6",
"xtend": "~4.0.1"
}
},
"node_modules/pend": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
@ -10345,6 +10425,19 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/process-streams": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/process-streams/-/process-streams-1.0.3.tgz",
"integrity": "sha512-xkIaM5vYnyekB88WyET78YEqXiaJRy0xcvIdE22n+myhvBT7LlLmX6iAtq7jDvVH8CUx2rqQsd32JdRyJMV3NA==",
"funding": [
"https://www.paypal.com/donate/?hosted_button_id=GB656ZSAEQEXN",
"https://de.liberapay.com/nils.knappmeier/"
],
"license": "MIT",
"dependencies": {
"duplex-maker": "^1.0.0"
}
},
"node_modules/process-warning": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz",
@ -11576,6 +11669,18 @@
"simple-concat": "^1.0.0"
}
},
"node_modules/simple-zstd": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/simple-zstd/-/simple-zstd-1.4.2.tgz",
"integrity": "sha512-kGYEvT33M5XfyQvvW4wxl3eKcWbdbCc1V7OZzuElnaXft0qbVzoIIXHXiCm3JCUki+MZKKmvjl8p2VGLJc5Y/A==",
"license": "MIT",
"dependencies": {
"is-zst": "^1.0.0",
"peek-stream": "^1.1.3",
"process-streams": "^1.0.1",
"through2": "^4.0.2"
}
},
"node_modules/sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
@ -12932,6 +13037,15 @@
"node": ">=4.0"
}
},
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"license": "MIT",
"engines": {
"node": ">=0.4"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

1
backend/functions/package.json
View File

@ -60,6 +60,7 @@
"puppeteer-page-proxy": "^1.3.0",
"robots-parser": "^3.0.1",
"set-cookie-parser": "^2.6.0",
"simple-zstd": "^1.4.2",
"stripe": "^11.11.0",
"tiktoken": "^1.0.16",
"tld-extract": "^2.1.0",

9
backend/functions/src/services/curl.ts
View File

@ -11,6 +11,7 @@ import { TempFileManager } from '../shared';
import { readFile } from 'fs/promises';
import { pathToFileURL } from 'url';
import { createBrotliDecompress, createInflate, createGunzip } from 'zlib';
import { ZSTDDecompress } from 'simple-zstd';
@singleton()
export class CurlControl extends AsyncService {
@ -43,7 +44,7 @@ export class CurlControl extends AsyncService {
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Encoding': 'gzip, deflate, br, zstd',
'Accept-Language': 'en-US,en;q=0.9',
};
@ -157,6 +158,12 @@ export class CurlControl extends AsyncService {
stream = decompressed;
break;
}
case 'zstd': {
const decompressed = ZSTDDecompress();
stream.pipe(decompressed);
stream = decompressed;
break;
}
default: {
break;
}

9
backend/functions/src/types.d.ts vendored
View File

@ -14,8 +14,15 @@ declare module 'jsdom' {
constructor(html: string, options?: any);
window: typeof window;
}
export class VirtualConsole extends EventEmitter{
export class VirtualConsole extends EventEmitter {
constructor();
sendTo(console: any, options?: any);
}
}
declare module 'simple-zstd' {
import { Duplex } from 'stream';
export function ZSTDCompress(lvl: Number): Duplex;
export function ZSTDDecompress(): Duplex;
export function ZSTDDecompressMaybe(): Duplex;
}