From b083d2d8401e0080aa418a47a91d0e1a51e7d9bf Mon Sep 17 00:00:00 2001 From: xream Date: Sun, 12 May 2024 23:17:11 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20Node.js=20=E7=89=88=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20MMDB,=20=E9=80=9A=E8=BF=87=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E6=88=96=E5=9C=A8=E8=84=9A=E6=9C=AC=E4=B8=AD?= =?UTF-8?q?=E4=BC=A0=E5=85=A5=E6=95=B0=E6=8D=AE=E5=BA=93=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=B7=AF=E5=BE=84,=20=E5=8F=AF=E4=BD=BF=E7=94=A8=20ipaso=20?= =?UTF-8?q?=E5=92=8C=20geoip=20=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/package.json | 3 +- backend/pnpm-lock.yaml | 63 +++++++++++++++++++++++++++ backend/src/core/proxy-utils/index.js | 3 +- backend/src/utils/geo.js | 30 +++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/backend/package.json b/backend/package.json index a5989bd..cc8dfa7 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "sub-store", - "version": "2.14.308", + "version": "2.14.309", "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.", "main": "src/main.js", "scripts": { @@ -17,6 +17,7 @@ "author": "Peng-YM", "license": "GPL-3.0", "dependencies": { + "@maxmind/geoip2-node": "^5.0.0", "automerge": "1.0.1-preview.7", "body-parser": "^1.19.0", "connect-history-api-fallback": "^2.0.0", diff --git a/backend/pnpm-lock.yaml b/backend/pnpm-lock.yaml index b2df069..9fcd7ff 100644 --- a/backend/pnpm-lock.yaml +++ b/backend/pnpm-lock.yaml @@ -5,6 +5,9 @@ settings: excludeLinksFromLockfile: false dependencies: + '@maxmind/geoip2-node': + specifier: ^5.0.0 + version: registry.npmmirror.com/@maxmind/geoip2-node@5.0.0 automerge: specifier: 1.0.1-preview.7 version: registry.npmmirror.com/automerge@1.0.1-preview.7 @@ -1967,6 +1970,15 @@ packages: '@jridgewell/sourcemap-codec': registry.npmmirror.com/@jridgewell/sourcemap-codec@1.4.13 dev: true + registry.npmmirror.com/@maxmind/geoip2-node@5.0.0: + resolution: {integrity: sha512-ki+q5//oU4tZ3BAhegZJcB5czoZyic5JSTEKbrUAQB/BzAoAiGyLW0immEmQvVVyy2SMlvBTJ3zqyRj8K9BbwQ==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@maxmind/geoip2-node/-/geoip2-node-5.0.0.tgz} + name: '@maxmind/geoip2-node' + version: 5.0.0 + dependencies: + ip6addr: registry.npmmirror.com/ip6addr@0.2.5 + maxmind: registry.npmmirror.com/maxmind@4.3.19 + dev: false + registry.npmmirror.com/@sindresorhus/is@0.14.0: resolution: {integrity: sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@sindresorhus/is/-/is-0.14.0.tgz} name: '@sindresorhus/is' @@ -5988,6 +6000,15 @@ packages: engines: {node: '>=0.10.0'} dev: true + registry.npmmirror.com/ip6addr@0.2.5: + resolution: {integrity: sha512-9RGGSB6Zc9Ox5DpDGFnJdIeF0AsqXzdH+FspCfPPaU/L/4tI6P+5lIoFUFm9JXs9IrJv1boqAaNCQmoDADTSKQ==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ip6addr/-/ip6addr-0.2.5.tgz} + name: ip6addr + version: 0.2.5 + dependencies: + assert-plus: registry.npmmirror.com/assert-plus@1.0.0 + jsprim: registry.npmmirror.com/jsprim@2.0.2 + dev: false + registry.npmmirror.com/ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz} name: ipaddr.js @@ -6556,6 +6577,12 @@ packages: version: 0.2.3 dev: false + registry.npmmirror.com/json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/json-schema/-/json-schema-0.4.0.tgz} + name: json-schema + version: 0.4.0 + dev: false + registry.npmmirror.com/json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz} name: json-stable-stringify-without-jsonify @@ -6595,6 +6622,18 @@ packages: verror: registry.npmmirror.com/verror@1.10.0 dev: false + registry.npmmirror.com/jsprim@2.0.2: + resolution: {integrity: sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsprim/-/jsprim-2.0.2.tgz} + name: jsprim + version: 2.0.2 + engines: {'0': node >=0.6.0} + dependencies: + assert-plus: registry.npmmirror.com/assert-plus@1.0.0 + extsprintf: registry.npmmirror.com/extsprintf@1.3.0 + json-schema: registry.npmmirror.com/json-schema@0.4.0 + verror: registry.npmmirror.com/verror@1.10.0 + dev: false + registry.npmmirror.com/just-debounce@1.1.0: resolution: {integrity: sha512-qpcRocdkUmf+UTNBYx5w6dexX5J31AKK1OmPwH630a83DdVVUIngk55RSAiIGpQyoH0dlr872VHfPjnQnK1qDQ==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/just-debounce/-/just-debounce-1.1.0.tgz} name: just-debounce @@ -6943,6 +6982,16 @@ packages: - supports-color dev: true + registry.npmmirror.com/maxmind@4.3.19: + resolution: {integrity: sha512-Bu/VEN7ZWAOCjifdZaXJQuN6/yO7+OK35pnJsqmz8sOndK3KQFvJoY+6HX09/MmLLqtCfa+sMK0iaQOaTejGNA==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/maxmind/-/maxmind-4.3.19.tgz} + name: maxmind + version: 4.3.19 + engines: {node: '>=12', npm: '>=6'} + dependencies: + mmdb-lib: registry.npmmirror.com/mmdb-lib@2.1.0 + tiny-lru: registry.npmmirror.com/tiny-lru@11.2.6 + dev: false + registry.npmmirror.com/md5.js@1.3.5: resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/md5.js/-/md5.js-1.3.5.tgz} name: md5.js @@ -7132,6 +7181,13 @@ packages: version: 0.5.3 dev: true + registry.npmmirror.com/mmdb-lib@2.1.0: + resolution: {integrity: sha512-tdDTZmnI5G4UoSctv2KxM/3VQt2XRj4CmR5R4VsAWsOUcS3LysHR34wtixWm/pXxXdkBDuN92auxkC0T2+qd1Q==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mmdb-lib/-/mmdb-lib-2.1.0.tgz} + name: mmdb-lib + version: 2.1.0 + engines: {node: '>=10', npm: '>=6'} + dev: false + registry.npmmirror.com/mocha@10.0.0: resolution: {integrity: sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mocha/-/mocha-10.0.0.tgz} name: mocha @@ -9424,6 +9480,13 @@ packages: process: registry.npmmirror.com/process@0.11.10 dev: true + registry.npmmirror.com/tiny-lru@11.2.6: + resolution: {integrity: sha512-0PU3c9PjMnltZaFo2sGYv/nnJsMjG0Cxx8X6FXHPPGjFyoo1SJDxvUXW1207rdiSxYizf31roo+GrkIByQeZoA==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tiny-lru/-/tiny-lru-11.2.6.tgz} + name: tiny-lru + version: 11.2.6 + engines: {node: '>=12'} + dev: false + registry.npmmirror.com/tinyify@3.0.0: resolution: {integrity: sha512-RtjVjC1xwwxt8AMVfxEmo+FzRJB6p5sAOtFaJj8vMrkWShtArsM4dLVRWhx2Vc07Me3NWgmP7pi9UPm/a2XNNA==, registry: http://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tinyify/-/tinyify-3.0.0.tgz} name: tinyify diff --git a/backend/src/core/proxy-utils/index.js b/backend/src/core/proxy-utils/index.js index 6a0aca5..4180273 100644 --- a/backend/src/core/proxy-utils/index.js +++ b/backend/src/core/proxy-utils/index.js @@ -15,7 +15,7 @@ import $ from '@/core/app'; import { FILES_KEY, MODULES_KEY } from '@/constants'; import { findByName } from '@/utils/database'; import { produceArtifact } from '@/restful/sync'; -import { getFlag, getISO } from '@/utils/geo'; +import { getFlag, getISO, MMDB } from '@/utils/geo'; import Gist from '@/utils/gist'; function preprocess(raw) { @@ -273,6 +273,7 @@ export const ProxyUtils = { yaml: YAML, getFlag, getISO, + MMDB, Gist, }; diff --git a/backend/src/utils/geo.js b/backend/src/utils/geo.js index 28c17b8..29b222c 100644 --- a/backend/src/utils/geo.js +++ b/backend/src/utils/geo.js @@ -1,3 +1,5 @@ +import $ from '@/core/app'; + const ISOFlags = { '🏳️‍🌈': ['EXP', 'BAND'], '🇸🇱': ['TEST', 'SOS'], @@ -427,3 +429,31 @@ export function getFlag(name) { export function getISO(name) { return ISOFlags[getFlag(name)]?.[0]; } + +export class MMDB { + constructor({ country, asn } = {}) { + if ($.env.isNode) { + const Reader = eval(`require("@maxmind/geoip2-node")`).Reader; + const fs = eval("require('fs')"); + const countryFile = + country || eval('process.env.SUB_STORE_MMDB_COUNTRY_PATH'); + const asnFile = asn || eval('process.env.SUB_STORE_MMDB_ASN_PATH'); + if (countryFile) { + this.countryReader = Reader.openBuffer( + fs.readFileSync(countryFile), + ); + } + if (asnFile) { + if (!fs.existsSync(asnFile)) + throw new Error('GeoLite2 ASN MMDB does not exist'); + this.asnReader = Reader.openBuffer(fs.readFileSync(asnFile)); + } + } + } + geoip(ip) { + return this.countryReader?.country(ip)?.country?.isoCode; + } + ipaso(ip) { + return this.asnReader?.asn(ip)?.autonomousSystemOrganization; + } +} From e6d1aa1150680a1ac5934a2e5b16020f1df5fb47 Mon Sep 17 00:00:00 2001 From: xream Date: Mon, 13 May 2024 17:08:17 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat:=20=E4=BD=BF=E7=94=A8=E4=BA=86?= =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=E7=BC=93=E5=AD=98=20cacheKey=20?= =?UTF-8?q?=E7=9A=84=E8=BF=9C=E7=A8=8B=E8=AE=A2=E9=98=85=20=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E4=B8=BA=E4=B9=90=E8=A7=82=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/package.json | 2 +- backend/src/utils/download.js | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/backend/package.json b/backend/package.json index cc8dfa7..c52ee32 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "sub-store", - "version": "2.14.309", + "version": "2.14.310", "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.", "main": "src/main.js", "scripts": { diff --git a/backend/src/utils/download.js b/backend/src/utils/download.js index 12f4b2a..6328106 100644 --- a/backend/src/utils/download.js +++ b/backend/src/utils/download.js @@ -14,7 +14,13 @@ import $ from '@/core/app'; const tasks = new Map(); -export default async function download(rawUrl, ua, timeout, proxy) { +export default async function download( + rawUrl, + ua, + timeout, + proxy, + skipCustomCache, +) { let $arguments = {}; let url = rawUrl.replace(/#noFlow$/, ''); const rawArgs = url.split('#'); @@ -39,6 +45,23 @@ export default async function download(rawUrl, ua, timeout, proxy) { ? `#sub-store-cached-custom-${$arguments?.cacheKey}` : undefined; + if (customCacheKey && !skipCustomCache) { + const cached = $.read(customCacheKey); + if (cached) { + $.info( + `乐观缓存: URL ${url}\n本次返回自定义缓存 ${$arguments?.cacheKey}\n并进行请求 尝试更新缓存`, + ); + download( + rawUrl.replace(/(\?|&)cacheKey=.*?(&|$)/, ''), + ua, + timeout, + proxy, + true, + ); + return cached; + } + } + // const downloadUrlMatch = url.match(/^\/api\/(file|module)\/(.+)/); // if (downloadUrlMatch) { // let type = downloadUrlMatch?.[1]; @@ -81,7 +104,7 @@ export default async function download(rawUrl, ua, timeout, proxy) { // try to find in app cache const cached = resourceCache.get(id); - if (!$arguments?.noCache && cached) { + if (!$arguments?.noCache && cached && !skipCustomCache) { $.info(`使用缓存: ${url}`); result = cached; } else { @@ -120,6 +143,9 @@ export default async function download(rawUrl, ua, timeout, proxy) { if (shouldCache) { resourceCache.set(id, body); if (customCacheKey) { + $.info( + `URL ${url}\n写入自定义缓存 ${$arguments?.cacheKey}`, + ); $.write(body, customCacheKey); } }