From 5584225413f77bc9955162a5044f9a1fa4c37f3f Mon Sep 17 00:00:00 2001 From: xream Date: Sun, 14 Jan 2024 23:37:55 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E8=AE=A2=E9=98=85?= =?UTF-8?q?=E6=B5=81=E9=87=8F=E8=8E=B7=E5=8F=96,=20=E5=90=AF=E7=94=A8?= =?UTF-8?q?=E5=85=B1=E4=BA=AB=E7=BC=93=E5=AD=98(=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E4=B8=80=E5=88=86=E9=92=9F)=20=E5=B9=B6=E4=BC=98=E5=85=88?= =?UTF-8?q?=E5=B0=9D=E8=AF=95=20HEAD=20=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/package.json | 2 +- backend/src/constants.js | 2 + .../src/core/proxy-utils/processors/index.js | 14 ++- backend/src/utils/download.js | 10 +- backend/src/utils/flow.js | 90 ++++++++++++--- backend/src/utils/headers-resource-cache.js | 107 ++++++++++++++++++ 6 files changed, 203 insertions(+), 22 deletions(-) create mode 100644 backend/src/utils/headers-resource-cache.js diff --git a/backend/package.json b/backend/package.json index f7bf3d9..6345fe7 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "sub-store", - "version": "2.14.160", + "version": "2.14.161", "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.", "main": "src/main.js", "scripts": { diff --git a/backend/src/constants.js b/backend/src/constants.js index 7481357..3a5cd9b 100644 --- a/backend/src/constants.js +++ b/backend/src/constants.js @@ -10,6 +10,8 @@ export const GIST_BACKUP_KEY = 'Auto Generated Sub-Store Backup'; export const GIST_BACKUP_FILE_NAME = 'Sub-Store'; export const ARTIFACT_REPOSITORY_KEY = 'Sub-Store Artifacts Repository'; export const RESOURCE_CACHE_KEY = '#sub-store-cached-resource'; +export const HEADERS_RESOURCE_CACHE_KEY = '#sub-store-cached-headers-resource'; +export const CHR_EXPIRATION_TIME_KEY = '#sub-store-chr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 1 min export const CACHE_EXPIRATION_TIME_MS = 60 * 60 * 1000; // 1 hour export const SCRIPT_RESOURCE_CACHE_KEY = '#sub-store-cached-script-resource'; // cached-script-resource CSR export const CSR_EXPIRATION_TIME_KEY = '#sub-store-csr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 48 hour diff --git a/backend/src/core/proxy-utils/processors/index.js b/backend/src/core/proxy-utils/processors/index.js index 09e2cdd..75b5259 100644 --- a/backend/src/core/proxy-utils/processors/index.js +++ b/backend/src/core/proxy-utils/processors/index.js @@ -10,7 +10,12 @@ import { ProxyUtils } from '@/core/proxy-utils'; import { produceArtifact } from '@/restful/sync'; import env from '@/utils/env'; -import { getFlowHeaders, parseFlowHeaders, flowTransfer } from '@/utils/flow'; +import { + getFlowField, + getFlowHeaders, + parseFlowHeaders, + flowTransfer, +} from '@/utils/flow'; /** The rule "(name CONTAINS "🇨🇳") AND (port IN [80, 443])" can be expressed as follows: @@ -781,7 +786,12 @@ function removeFlag(str) { } function createDynamicFunction(name, script, $arguments) { - const flowUtils = { getFlowHeaders, parseFlowHeaders, flowTransfer }; + const flowUtils = { + getFlowField, + getFlowHeaders, + parseFlowHeaders, + flowTransfer, + }; if ($.env.isLoon) { return new Function( '$arguments', diff --git a/backend/src/utils/download.js b/backend/src/utils/download.js index 5548385..5274c72 100644 --- a/backend/src/utils/download.js +++ b/backend/src/utils/download.js @@ -3,6 +3,8 @@ import { findByName } from '@/utils/database'; import { HTTP, ENV } from '@/vendor/open-api'; import { hex_md5 } from '@/vendor/md5'; import resourceCache from '@/utils/resource-cache'; +import headersResourceCache from '@/utils/headers-resource-cache'; +import { getFlowField } from '@/utils/flow'; import $ from '@/core/app'; const tasks = new Map(); @@ -71,7 +73,13 @@ export default async function download(url, ua, timeout) { ); http.get(url) .then((resp) => { - const body = resp.body; + const { body, headers } = resp; + if (headers) { + const flowInfo = getFlowField(headers); + if (flowInfo) { + headersResourceCache.set(url, flowInfo); + } + } if (body.replace(/\s/g, '').length === 0) reject(new Error('远程资源内容为空！')); else { diff --git a/backend/src/utils/flow.js b/backend/src/utils/flow.js index cdf2bfc..9391fa9 100644 --- a/backend/src/utils/flow.js +++ b/backend/src/utils/flow.js @@ -1,30 +1,84 @@ import { SETTINGS_KEY } from '@/constants'; import { HTTP } from '@/vendor/open-api'; import $ from '@/core/app'; +import headersResourceCache from '@/utils/headers-resource-cache'; -export async function getFlowHeaders(url, ua, timeout) { - const { defaultFlowUserAgent, defaultTimeout } = $.read(SETTINGS_KEY); - const userAgent = - ua || - defaultFlowUserAgent || - 'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)'; - const requestTimeout = timeout || defaultTimeout; - const http = HTTP(); - const { headers } = await http.get({ - url: url - .split(/[\r\n]+/) - .map((i) => i.trim()) - .filter((i) => i.length)[0], - headers: { - 'User-Agent': userAgent, - }, - timeout: requestTimeout, - }); +export function getFlowField(headers) { const subkey = Object.keys(headers).filter((k) => /SUBSCRIPTION-USERINFO/i.test(k), )[0]; return headers[subkey]; } +export async function getFlowHeaders(url, ua, timeout) { + let $arguments = {}; + const rawArgs = url.split('#'); + if (rawArgs.length > 1) { + try { + // 支持 `#${encodeURIComponent(JSON.stringify({arg1: "1"}))}` + $arguments = JSON.parse(decodeURIComponent(rawArgs[1])); + } catch (e) { + for (const pair of rawArgs[1].split('&')) { + const key = pair.split('=')[0]; + const value = pair.split('=')[1]; + // 部分兼容之前的逻辑 const value = pair.split('=')[1] || true; + $arguments[key] = + value == null || value === '' + ? true + : decodeURIComponent(value); + } + } + } + const cached = headersResourceCache.get(url); + let flowInfo; + if (!$arguments?.noCache && cached) { + $.info(`使用缓存的流量信息: ${url}`); + flowInfo = cached; + } else { + const { defaultFlowUserAgent, defaultTimeout } = $.read(SETTINGS_KEY); + const userAgent = + ua || + defaultFlowUserAgent || + 'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)'; + const requestTimeout = timeout || defaultTimeout; + const http = HTTP(); + try { + $.info(`使用 HEAD 方法获取流量信息: ${url}`); + const { headers } = await http.head({ + url: url + .split(/[\r\n]+/) + .map((i) => i.trim()) + .filter((i) => i.length)[0], + headers: { + 'User-Agent': userAgent, + }, + timeout: requestTimeout, + }); + flowInfo = getFlowField(headers); + } catch (e) { + $.error( + `使用 HEAD 方法获取流量信息失败: ${url}: ${e.message ?? e}`, + ); + } + if (!flowInfo) { + $.info(`使用 GET 方法获取流量信息: ${url}`); + const { headers } = await http.get({ + url: url + .split(/[\r\n]+/) + .map((i) => i.trim()) + .filter((i) => i.length)[0], + headers: { + 'User-Agent': userAgent, + }, + timeout: requestTimeout, + }); + flowInfo = getFlowField(headers); + } + } + if (flowInfo) { + headersResourceCache.set(url, flowInfo); + } + return flowInfo; +} export function parseFlowHeaders(flowHeaders) { if (!flowHeaders) return; // unit is KB diff --git a/backend/src/utils/headers-resource-cache.js b/backend/src/utils/headers-resource-cache.js new file mode 100644 index 0000000..4a4fcbc --- /dev/null +++ b/backend/src/utils/headers-resource-cache.js @@ -0,0 +1,107 @@ +import $ from '@/core/app'; +import { + HEADERS_RESOURCE_CACHE_KEY, + CHR_EXPIRATION_TIME_KEY, +} from '@/constants'; + +class ResourceCache { + constructor() { + this.expires = getExpiredTime(); + if (!$.read(HEADERS_RESOURCE_CACHE_KEY)) { + $.write('{}', HEADERS_RESOURCE_CACHE_KEY); + } + this.resourceCache = JSON.parse($.read(HEADERS_RESOURCE_CACHE_KEY)); + this._cleanup(); + } + + _cleanup() { + // clear obsolete cached resource + let clear = false; + Object.entries(this.resourceCache).forEach((entry) => { + const [id, updated] = entry; + if (!updated.time) { + // clear old version cache + delete this.resourceCache[id]; + $.delete(`#${id}`); + clear = true; + } + if (new Date().getTime() - updated.time > this.expires) { + delete this.resourceCache[id]; + clear = true; + } + }); + if (clear) this._persist(); + } + + revokeAll() { + this.resourceCache = {}; + this._persist(); + } + + _persist() { + $.write(JSON.stringify(this.resourceCache), HEADERS_RESOURCE_CACHE_KEY); + } + + get(id) { + const updated = this.resourceCache[id] && this.resourceCache[id].time; + if (updated && new Date().getTime() - updated <= this.expires) { + return this.resourceCache[id].data; + } + return null; + } + + gettime(id) { + const updated = this.resourceCache[id] && this.resourceCache[id].time; + if (updated && new Date().getTime() - updated <= this.expires) { + return this.resourceCache[id].time; + } + return null; + } + + set(id, value) { + this.resourceCache[id] = { time: new Date().getTime(), data: value }; + this._persist(); + } +} + +function getExpiredTime() { + // console.log($.read(CHR_EXPIRATION_TIME_KEY)); + if (!$.read(CHR_EXPIRATION_TIME_KEY)) { + $.write('6e4', CHR_EXPIRATION_TIME_KEY); // 1分钟 + } + let expiration = 6e4; + if ($.env.isLoon) { + const loont = { + // Loon 插件自义定 + '1\u5206\u949f': 6e4, + '5\u5206\u949f': 3e5, + '10\u5206\u949f': 6e5, + '30\u5206\u949f': 18e5, // "30分钟" + '1\u5c0f\u65f6': 36e5, + '2\u5c0f\u65f6': 72e5, + '3\u5c0f\u65f6': 108e5, + '6\u5c0f\u65f6': 216e5, + '12\u5c0f\u65f6': 432e5, + '24\u5c0f\u65f6': 864e5, + '48\u5c0f\u65f6': 1728e5, + '72\u5c0f\u65f6': 2592e5, // "72小时" + '\u53c2\u6570\u4f20\u5165': 'readcachets', // "参数输入" + }; + let intimed = $.read( + '#\u54cd\u5e94\u5934\u7f13\u5b58\u6709\u6548\u671f', + ); // Loon #响应头缓存有效期 + // console.log(intimed); + if (intimed in loont) { + expiration = loont[intimed]; + if (expiration === 'readcachets') { + expiration = intimed; + } + } + return expiration; + } else { + expiration = $.read(CHR_EXPIRATION_TIME_KEY); + return expiration; + } +} + +export default new ResourceCache();