From b6846ab4b62cd7367a58e6a7b343f8a45be16861 Mon Sep 17 00:00:00 2001 From: "yanlong.wang" Date: Wed, 19 Feb 2025 14:22:19 +0800 Subject: [PATCH] curl-impersonate: add zstd compression options --- backend/functions/Dockerfile | 2 +- backend/functions/package-lock.json | 120 ++++++++++++++++++++++++- backend/functions/package.json | 1 + backend/functions/src/services/curl.ts | 9 +- backend/functions/src/types.d.ts | 9 +- 5 files changed, 135 insertions(+), 6 deletions(-) diff --git a/backend/functions/Dockerfile b/backend/functions/Dockerfile index 6d5f610..c63914f 100644 --- a/backend/functions/Dockerfile +++ b/backend/functions/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ && apt-get update \ - && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ + && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* diff --git a/backend/functions/package-lock.json b/backend/functions/package-lock.json index f90044d..ea7ed5e 100644 --- a/backend/functions/package-lock.json +++ b/backend/functions/package-lock.json @@ -40,6 +40,7 @@ "puppeteer-page-proxy": "^1.3.0", "robots-parser": "^3.0.1", "set-cookie-parser": "^2.6.0", + "simple-zstd": "^1.4.2", "stripe": "^11.11.0", "tiktoken": "^1.0.16", "tld-extract": "^2.1.0", @@ -3701,9 +3702,7 @@ "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, - "peer": true + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, "node_modules/busboy": { "version": "1.6.0", @@ -4761,6 +4760,12 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/duplex-maker": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/duplex-maker/-/duplex-maker-1.0.0.tgz", + "integrity": "sha512-KoHuzggxg7f+vvjqOHfXxaQYI1POzBm+ah0eec7YDssZmbt6QFBI8d1nl5GQwAgR2f+VQCPvyvZtmWWqWuFtlA==", + "license": "MIT" + }, "node_modules/duplexify": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", @@ -7288,6 +7293,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-zst": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-zst/-/is-zst-1.0.0.tgz", + "integrity": "sha512-ZA5lvshKAl8z30dX7saXLpVhpsq3d2EHK9uf7qtUjnOtdw4XBpAoWb2RvZ5kyoaebdoidnGI0g2hn9Z7ObPbww==", + "license": "MIT" + }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -10043,6 +10054,75 @@ "path2d": "^0.2.0" } }, + "node_modules/peek-stream": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", + "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "duplexify": "^3.5.0", + "through2": "^2.0.3" + } + }, + "node_modules/peek-stream/node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/peek-stream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/peek-stream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/peek-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/peek-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/peek-stream/node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "license": "MIT", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -10345,6 +10425,19 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/process-streams": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/process-streams/-/process-streams-1.0.3.tgz", + "integrity": "sha512-xkIaM5vYnyekB88WyET78YEqXiaJRy0xcvIdE22n+myhvBT7LlLmX6iAtq7jDvVH8CUx2rqQsd32JdRyJMV3NA==", + "funding": [ + "https://www.paypal.com/donate/?hosted_button_id=GB656ZSAEQEXN", + "https://de.liberapay.com/nils.knappmeier/" + ], + "license": "MIT", + "dependencies": { + "duplex-maker": "^1.0.0" + } + }, "node_modules/process-warning": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", @@ -11576,6 +11669,18 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-zstd": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/simple-zstd/-/simple-zstd-1.4.2.tgz", + "integrity": "sha512-kGYEvT33M5XfyQvvW4wxl3eKcWbdbCc1V7OZzuElnaXft0qbVzoIIXHXiCm3JCUki+MZKKmvjl8p2VGLJc5Y/A==", + "license": "MIT", + "dependencies": { + "is-zst": "^1.0.0", + "peek-stream": "^1.1.3", + "process-streams": "^1.0.1", + "through2": "^4.0.2" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -12932,6 +13037,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/backend/functions/package.json b/backend/functions/package.json index 07c2714..a44f86c 100644 --- a/backend/functions/package.json +++ b/backend/functions/package.json @@ -60,6 +60,7 @@ "puppeteer-page-proxy": "^1.3.0", "robots-parser": "^3.0.1", "set-cookie-parser": "^2.6.0", + "simple-zstd": "^1.4.2", "stripe": "^11.11.0", "tiktoken": "^1.0.16", "tld-extract": "^2.1.0", diff --git a/backend/functions/src/services/curl.ts b/backend/functions/src/services/curl.ts index 2697a84..6bb0088 100644 --- a/backend/functions/src/services/curl.ts +++ b/backend/functions/src/services/curl.ts @@ -11,6 +11,7 @@ import { TempFileManager } from '../shared'; import { readFile } from 'fs/promises'; import { pathToFileURL } from 'url'; import { createBrotliDecompress, createInflate, createGunzip } from 'zlib'; +import { ZSTDDecompress } from 'simple-zstd'; @singleton() export class CurlControl extends AsyncService { @@ -43,7 +44,7 @@ export class CurlControl extends AsyncService { 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-User': '?1', 'Sec-Fetch-Dest': 'document', - 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept-Language': 'en-US,en;q=0.9', }; @@ -157,6 +158,12 @@ export class CurlControl extends AsyncService { stream = decompressed; break; } + case 'zstd': { + const decompressed = ZSTDDecompress(); + stream.pipe(decompressed); + stream = decompressed; + break; + } default: { break; } diff --git a/backend/functions/src/types.d.ts b/backend/functions/src/types.d.ts index 7919e03..b2ab9ce 100644 --- a/backend/functions/src/types.d.ts +++ b/backend/functions/src/types.d.ts @@ -14,8 +14,15 @@ declare module 'jsdom' { constructor(html: string, options?: any); window: typeof window; } - export class VirtualConsole extends EventEmitter{ + export class VirtualConsole extends EventEmitter { constructor(); sendTo(console: any, options?: any); } } + +declare module 'simple-zstd' { + import { Duplex } from 'stream'; + export function ZSTDCompress(lvl: Number): Duplex; + export function ZSTDDecompress(): Duplex; + export function ZSTDDecompressMaybe(): Duplex; +}