Local-GitHub-Proxy/Sub-Store
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/sub-store-org/Sub-Store.git synced 2025-08-10 09:19:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 优化订阅流量获取, 启用共享缓存(默认一分钟) 并优先尝试 HEAD 方法

Browse Source
This commit is contained in:
xream 2024-01-14 23:37:55 +08:00
parent 5cbcf4fce4
commit 5584225413
No known key found for this signature in database
GPG Key ID: 1D2C5225471789F9
6 changed files with 203 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/package.json
View File

@ -1,6 +1,6 @@
{
"name": "sub-store",
"version": "2.14.160",
"version": "2.14.161",
"description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
"main": "src/main.js",
"scripts": {

2
backend/src/constants.js
View File

@ -10,6 +10,8 @@ export const GIST_BACKUP_KEY = 'Auto Generated Sub-Store Backup';
export const GIST_BACKUP_FILE_NAME = 'Sub-Store';
export const ARTIFACT_REPOSITORY_KEY = 'Sub-Store Artifacts Repository';
export const RESOURCE_CACHE_KEY = '#sub-store-cached-resource';
export const HEADERS_RESOURCE_CACHE_KEY = '#sub-store-cached-headers-resource';
export const CHR_EXPIRATION_TIME_KEY = '#sub-store-chr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 1 min
export const CACHE_EXPIRATION_TIME_MS = 60 * 60 * 1000; // 1 hour
export const SCRIPT_RESOURCE_CACHE_KEY = '#sub-store-cached-script-resource'; // cached-script-resource CSR
export const CSR_EXPIRATION_TIME_KEY = '#sub-store-csr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 48 hour

14
backend/src/core/proxy-utils/processors/index.js
View File

@ -10,7 +10,12 @@ import { ProxyUtils } from '@/core/proxy-utils';
import { produceArtifact } from '@/restful/sync';
import env from '@/utils/env';
import { getFlowHeaders, parseFlowHeaders, flowTransfer } from '@/utils/flow';
import {
getFlowField,
getFlowHeaders,
parseFlowHeaders,
flowTransfer,
} from '@/utils/flow';
/**
The rule "(name CONTAINS "🇨🇳") AND (port IN [80, 443])" can be expressed as follows:
@ -781,7 +786,12 @@ function removeFlag(str) {
}
function createDynamicFunction(name, script, $arguments) {
const flowUtils = { getFlowHeaders, parseFlowHeaders, flowTransfer };
const flowUtils = {
getFlowField,
getFlowHeaders,
parseFlowHeaders,
flowTransfer,
};
if ($.env.isLoon) {
return new Function(
'$arguments',

10
backend/src/utils/download.js
View File

@ -3,6 +3,8 @@ import { findByName } from '@/utils/database';
import { HTTP, ENV } from '@/vendor/open-api';
import { hex_md5 } from '@/vendor/md5';
import resourceCache from '@/utils/resource-cache';
import headersResourceCache from '@/utils/headers-resource-cache';
import { getFlowField } from '@/utils/flow';
import $ from '@/core/app';
const tasks = new Map();
@ -71,7 +73,13 @@ export default async function download(url, ua, timeout) {
);
http.get(url)
.then((resp) => {
const body = resp.body;
const { body, headers } = resp;
if (headers) {
const flowInfo = getFlowField(headers);
if (flowInfo) {
headersResourceCache.set(url, flowInfo);
}
}
if (body.replace(/\s/g, '').length === 0)
reject(new Error('远程资源内容为空！'));
else {

90
backend/src/utils/flow.js
View File

@ -1,30 +1,84 @@
import { SETTINGS_KEY } from '@/constants';
import { HTTP } from '@/vendor/open-api';
import $ from '@/core/app';
import headersResourceCache from '@/utils/headers-resource-cache';
export async function getFlowHeaders(url, ua, timeout) {
const { defaultFlowUserAgent, defaultTimeout } = $.read(SETTINGS_KEY);
const userAgent =
ua ||
defaultFlowUserAgent ||
'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)';
const requestTimeout = timeout || defaultTimeout;
const http = HTTP();
const { headers } = await http.get({
url: url
.split(/[\r\n]+/)
.map((i) => i.trim())
.filter((i) => i.length)[0],
headers: {
'User-Agent': userAgent,
},
timeout: requestTimeout,
});
export function getFlowField(headers) {
const subkey = Object.keys(headers).filter((k) =>
/SUBSCRIPTION-USERINFO/i.test(k),
)[0];
return headers[subkey];
}
export async function getFlowHeaders(url, ua, timeout) {
let $arguments = {};
const rawArgs = url.split('#');
if (rawArgs.length > 1) {
try {
// 支持 `#${encodeURIComponent(JSON.stringify({arg1: "1"}))}`
$arguments = JSON.parse(decodeURIComponent(rawArgs[1]));
} catch (e) {
for (const pair of rawArgs[1].split('&')) {
const key = pair.split('=')[0];
const value = pair.split('=')[1];
// 部分兼容之前的逻辑 const value = pair.split('=')[1] || true;
$arguments[key] =
value == null || value === ''
? true
: decodeURIComponent(value);
}
}
}
const cached = headersResourceCache.get(url);
let flowInfo;
if (!$arguments?.noCache && cached) {
$.info(`使用缓存的流量信息: ${url}`);
flowInfo = cached;
} else {
const { defaultFlowUserAgent, defaultTimeout } = $.read(SETTINGS_KEY);
const userAgent =
ua ||
defaultFlowUserAgent ||
'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)';
const requestTimeout = timeout || defaultTimeout;
const http = HTTP();
try {
$.info(`使用 HEAD 方法获取流量信息: ${url}`);
const { headers } = await http.head({
url: url
.split(/[\r\n]+/)
.map((i) => i.trim())
.filter((i) => i.length)[0],
headers: {
'User-Agent': userAgent,
},
timeout: requestTimeout,
});
flowInfo = getFlowField(headers);
} catch (e) {
$.error(
`使用 HEAD 方法获取流量信息失败: ${url}: ${e.message ?? e}`,
);
}
if (!flowInfo) {
$.info(`使用 GET 方法获取流量信息: ${url}`);
const { headers } = await http.get({
url: url
.split(/[\r\n]+/)
.map((i) => i.trim())
.filter((i) => i.length)[0],
headers: {
'User-Agent': userAgent,
},
timeout: requestTimeout,
});
flowInfo = getFlowField(headers);
}
}
if (flowInfo) {
headersResourceCache.set(url, flowInfo);
}
return flowInfo;
}
export function parseFlowHeaders(flowHeaders) {
if (!flowHeaders) return;
// unit is KB

107
backend/src/utils/headers-resource-cache.js Normal file
View File

@ -0,0 +1,107 @@
import $ from '@/core/app';
import {
HEADERS_RESOURCE_CACHE_KEY,
CHR_EXPIRATION_TIME_KEY,
} from '@/constants';
class ResourceCache {
constructor() {
this.expires = getExpiredTime();
if (!$.read(HEADERS_RESOURCE_CACHE_KEY)) {
$.write('{}', HEADERS_RESOURCE_CACHE_KEY);
}
this.resourceCache = JSON.parse($.read(HEADERS_RESOURCE_CACHE_KEY));
this._cleanup();
}
_cleanup() {
// clear obsolete cached resource
let clear = false;
Object.entries(this.resourceCache).forEach((entry) => {
const [id, updated] = entry;
if (!updated.time) {
// clear old version cache
delete this.resourceCache[id];
$.delete(`#${id}`);
clear = true;
}
if (new Date().getTime() - updated.time > this.expires) {
delete this.resourceCache[id];
clear = true;
}
});
if (clear) this._persist();
}
revokeAll() {
this.resourceCache = {};
this._persist();
}
_persist() {
$.write(JSON.stringify(this.resourceCache), HEADERS_RESOURCE_CACHE_KEY);
}
get(id) {
const updated = this.resourceCache[id] && this.resourceCache[id].time;
if (updated && new Date().getTime() - updated <= this.expires) {
return this.resourceCache[id].data;
}
return null;
}
gettime(id) {
const updated = this.resourceCache[id] && this.resourceCache[id].time;
if (updated && new Date().getTime() - updated <= this.expires) {
return this.resourceCache[id].time;
}
return null;
}
set(id, value) {
this.resourceCache[id] = { time: new Date().getTime(), data: value };
this._persist();
}
}
function getExpiredTime() {
// console.log($.read(CHR_EXPIRATION_TIME_KEY));
if (!$.read(CHR_EXPIRATION_TIME_KEY)) {
$.write('6e4', CHR_EXPIRATION_TIME_KEY); // 1分钟
}
let expiration = 6e4;
if ($.env.isLoon) {
const loont = {
// Loon 插件自义定
'1\u5206\u949f': 6e4,
'5\u5206\u949f': 3e5,
'10\u5206\u949f': 6e5,
'30\u5206\u949f': 18e5, // "30分钟"
'1\u5c0f\u65f6': 36e5,
'2\u5c0f\u65f6': 72e5,
'3\u5c0f\u65f6': 108e5,
'6\u5c0f\u65f6': 216e5,
'12\u5c0f\u65f6': 432e5,
'24\u5c0f\u65f6': 864e5,
'48\u5c0f\u65f6': 1728e5,
'72\u5c0f\u65f6': 2592e5, // "72小时"
'\u53c2\u6570\u4f20\u5165': 'readcachets', // "参数输入"
};
let intimed = $.read(
'#\u54cd\u5e94\u5934\u7f13\u5b58\u6709\u6548\u671f',
); // Loon #响应头缓存有效期
// console.log(intimed);
if (intimed in loont) {
expiration = loont[intimed];
if (expiration === 'readcachets') {
expiration = intimed;
}
}
return expiration;
} else {
expiration = $.read(CHR_EXPIRATION_TIME_KEY);
return expiration;
}
}
export default new ResourceCache();