fix: 默认 User-Agent 改为 clash.meta 后, 调整订阅预处理器的逻辑, 减少 Base64 误判

.github/workflows/main.yml

@@ -44,7 +44,7 @@ jobs:
         run: |
           cd backend
           SUBSTORE_RELEASE=`node --eval="process.stdout.write(require('./package.json').version)"`
-          echo "::set-output name=release_tag::$SUBSTORE_RELEASE"
+          echo "release_tag=$SUBSTORE_RELEASE" >> $GITHUB_OUTPUT
       - name: Prepare release
         run: |
           cd backend

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.14.98",
+  "version": "2.14.111",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
   "main": "src/main.js",
   "scripts": {

backend/src/core/proxy-utils/parsers/index.js

@@ -33,7 +33,9 @@ function URI_SS() {
         const serverAndPort = serverAndPortArray[1];
         const portIdx = serverAndPort.lastIndexOf(':');
         proxy.server = serverAndPort.substring(0, portIdx);
-        proxy.port = serverAndPort.substring(portIdx + 1);
+        proxy.port = `${serverAndPort.substring(portIdx + 1)}`.match(
+            /\d+/,
+        )?.[0];
 
         const userInfo = userInfoStr.split(':');
         proxy.cipher = userInfo[0];

backend/src/core/proxy-utils/parsers/peggy/loon.js

@@ -44,7 +44,7 @@ shadowsocksr = tag equals "shadowsocksr"i address method password (ssr_protocol/
     // handle ssr obfs
     proxy.obfs = obfs.type;
 }
-shadowsocks = tag equals "shadowsocks"i address method password (obfs_ss/obfs_host/obfs_uri/fast_open/udp_relay/others)* {
+shadowsocks = tag equals "shadowsocks"i address method password (obfs_typev obfs_hostv)? (obfs_ss/obfs_host/obfs_uri/fast_open/udp_relay/others)* {
     proxy.type = "ss";
     // handle ss obfs
     if (obfs.type == "http" || obfs.type === "tls") {
@@ -145,6 +145,9 @@ username = & {
 password = comma '"' match:[^"]* '"' { proxy.password = match.join(""); }
 uuid = comma '"' match:[^"]+ '"' { proxy.uuid = match.join(""); }
 
+obfs_typev = comma type:("http"/"tls") { obfs.type = type; }
+obfs_hostv = comma match:[^,]+ { obfs.host = match.join(""); }
+
 obfs_ss = comma "obfs-name" equals type:("http"/"tls") { obfs.type = type; }
 
 obfs_ssr = comma "obfs" equals type:("plain"/"http_simple"/"http_post"/"random_head"/"tls1.2_ticket_auth"/"tls1.2_ticket_fastauth") { obfs.type = type; }

backend/src/core/proxy-utils/parsers/peggy/loon.peg

@@ -42,7 +42,7 @@ shadowsocksr = tag equals "shadowsocksr"i address method password (ssr_protocol/
     // handle ssr obfs
     proxy.obfs = obfs.type;
 }
-shadowsocks = tag equals "shadowsocks"i address method password (obfs_ss/obfs_host/obfs_uri/fast_open/udp_relay/others)* {
+shadowsocks = tag equals "shadowsocks"i address method password (obfs_typev obfs_hostv)? (obfs_ss/obfs_host/obfs_uri/fast_open/udp_relay/others)* {
     proxy.type = "ss";
     // handle ss obfs
     if (obfs.type == "http" || obfs.type === "tls") {
@@ -143,6 +143,9 @@ username = & {
 password = comma '"' match:[^"]* '"' { proxy.password = match.join(""); }
 uuid = comma '"' match:[^"]+ '"' { proxy.uuid = match.join(""); }
 
+obfs_typev = comma type:("http"/"tls") { obfs.type = type; }
+obfs_hostv = comma match:[^,]+ { obfs.host = match.join(""); }
+
 obfs_ss = comma "obfs-name" equals type:("http"/"tls") { obfs.type = type; }
 
 obfs_ssr = comma "obfs" equals type:("plain"/"http_simple"/"http_post"/"random_head"/"tls1.2_ticket_auth"/"tls1.2_ticket_fastauth") { obfs.type = type; }

backend/src/core/proxy-utils/preprocessors/index.js

@@ -25,7 +25,10 @@ function Base64Encoded() {
     ];
 
     const test = function (raw) {
-        return keys.some((k) => raw.indexOf(k) !== -1);
+        return (
+            !/^\w+:\/\/\w+/im.test(raw) &&
+            keys.some((k) => raw.indexOf(k) !== -1)
+        );
     };
     const parse = function (raw) {
         raw = Base64.decode(raw);
@@ -37,7 +40,9 @@ function Base64Encoded() {
 function Clash() {
     const name = 'Clash Pre-processor';
     const test = function (raw) {
-        return /proxies/.test(raw);
+        if (!/proxies/.test(raw)) return false;
+        const content = safeLoad(raw);
+        return content.proxies && Array.isArray(content.proxies);
     };
     const parse = function (raw) {
         // Clash YAML format
@@ -105,4 +110,4 @@ function FullConfig() {
     return { name, test, parse };
 }
 
-export default [HTML(), Base64Encoded(), Clash(), SSD(), FullConfig()];
+export default [HTML(), Clash(), Base64Encoded(), SSD(), FullConfig()];

backend/src/core/proxy-utils/processors/index.js

@@ -477,8 +477,15 @@ function ResolveDomainOperator({ provider }) {
                 }
                 await Promise.all(currentBatch);
             }
-            proxies.forEach((proxy) => {
-                proxy.server = results[proxy.server] || proxy.server;
+            proxies.forEach((p) => {
+                if (!p['no-resolve']) {
+                    if (results[p.server]) {
+                        p.server = results[p.server];
+                        p.resolved = true;
+                    } else {
+                        p.resolved = false;
+                    }
+                }
             });
 
             return proxies;

backend/src/restful/download.js

@@ -20,18 +20,22 @@ async function downloadSubscription(req, res) {
         req.query.target || getPlatformFromHeaders(req.headers) || 'JSON';
 
     $.info(`正在下载订阅：${name}`);
-    let { url, ua, content } = req.query;
+    let { url, ua, content, mergeSources } = req.query;
     if (url) {
         url = decodeURIComponent(url);
-        $.info(`指定 url: ${url}`);
+        $.info(`指定远程订阅 URL: ${url}`);
     }
     if (ua) {
         ua = decodeURIComponent(ua);
-        $.info(`指定 ua: ${ua}`);
+        $.info(`指定远程订阅 User-Agent: ${ua}`);
     }
     if (content) {
         content = decodeURIComponent(content);
-        $.info(`指定 content: ${content}`);
+        $.info(`指定本地订阅: ${content}`);
+    }
+    if (mergeSources) {
+        mergeSources = decodeURIComponent(mergeSources);
+        $.info(`指定合并来源: ${mergeSources}`);
     }
 
     const allSubs = $.read(SUBS_KEY);
@@ -45,6 +49,7 @@ async function downloadSubscription(req, res) {
                 url,
                 ua,
                 content,
+                mergeSources,
             });
 
             if (sub.source !== 'local' || url) {

backend/src/restful/index.js

@@ -15,8 +15,13 @@ import registerMiscRoutes from './miscs';
 import registerNodeInfoRoutes from './node-info';
 
 export default function serve() {
-    const $app = express({ substore: $ });
-
+    let port;
+    let host;
+    if ($.env.isNode) {
+        port = eval('process.env.SUB_STORE_BACKEND_API_PORT');
+        host = eval('process.env.SUB_STORE_BACKEND_API_HOST');
+    }
+    const $app = express({ substore: $, port, host });
     // register routes
     registerCollectionRoutes($app);
     registerSubscriptionRoutes($app);

backend/src/restful/miscs.js

@@ -22,12 +22,26 @@ export default function register($app) {
     // Storage management
     $app.route('/api/storage')
         .get((req, res) => {
-            res.json($.read('#sub-store'));
+            res.set('content-type', 'application/json')
+                .set(
+                    'content-disposition',
+                    'attachment; filename="sub-store.json"',
+                )
+                .send(
+                    $.env.isNode
+                        ? JSON.stringify($.cache)
+                        : $.read('#sub-store'),
+                );
         })
         .post((req, res) => {
-            const data = req.body;
-            $.write(JSON.stringify(data), '#sub-store');
-            res.end();
+            const { content } = req.body;
+            $.write(content, '#sub-store');
+            if ($.env.isNode) {
+                $.cache = JSON.parse(content);
+                $.persistCache();
+            }
+            migrate();
+            success(res);
         });
 
     // Redirect sub.store to vercel webpage

backend/src/restful/preview.js

@@ -16,11 +16,20 @@ async function compareSub(req, res) {
         const sub = req.body;
         const target = req.query.target || 'JSON';
         let content;
-        if (sub.source === 'local') {
+        if (
+            sub.source === 'local' &&
+            !['localFirst', 'remoteFirst'].includes(sub.mergeSources)
+        ) {
             content = sub.content;
         } else {
             try {
-                content = await download(sub.url, sub.ua);
+                content = await Promise.all(
+                    sub.url
+                        .split(/[\r\n]+/)
+                        .map((i) => i.trim())
+                        .filter((i) => i.length)
+                        .map((url) => download(url, sub.ua)),
+                );
             } catch (err) {
                 failed(
                     res,
@@ -32,9 +41,16 @@ async function compareSub(req, res) {
                 );
                 return;
             }
+            if (sub.mergeSources === 'localFirst') {
+                content.unshift(sub.content);
+            } else if (sub.mergeSources === 'remoteFirst') {
+                content.push(sub.content);
+            }
         }
         // parse proxies
-        const original = ProxyUtils.parse(content);
+        const original = (Array.isArray(content) ? content : [content])
+            .map((i) => ProxyUtils.parse(i))
+            .flat();
 
         // add id
         original.forEach((proxy, i) => {
@@ -77,13 +93,31 @@ async function compareCollection(req, res) {
                 const sub = findByName(allSubs, name);
                 try {
                     let raw;
-                    if (sub.source === 'local') {
+                    if (
+                        sub.source === 'local' &&
+                        !['localFirst', 'remoteFirst'].includes(
+                            sub.mergeSources,
+                        )
+                    ) {
                         raw = sub.content;
                     } else {
-                        raw = await download(sub.url, sub.ua);
+                        raw = await Promise.all(
+                            sub.url
+                                .split(/[\r\n]+/)
+                                .map((i) => i.trim())
+                                .filter((i) => i.length)
+                                .map((url) => download(url, sub.ua)),
+                        );
+                        if (sub.mergeSources === 'localFirst') {
+                            raw.unshift(sub.content);
+                        } else if (sub.mergeSources === 'remoteFirst') {
+                            raw.push(sub.content);
+                        }
                     }
                     // parse proxies
-                    let currentProxies = ProxyUtils.parse(raw);
+                    let currentProxies = (Array.isArray(raw) ? raw : [raw])
+                        .map((i) => ProxyUtils.parse(i))
+                        .flat();
 
                     currentProxies.forEach((proxy) => {
                         proxy.subName = sub.name;

backend/src/restful/settings.js

@@ -10,7 +10,12 @@ export default function register($app) {
 }
 
 async function getSettings(req, res) {
-    const settings = $.read(SETTINGS_KEY);
+    let settings = $.read(SETTINGS_KEY);
+    if (!settings) {
+        settings = {};
+        $.write(settings, SETTINGS_KEY);
+    }
+
     if (!settings.avatarUrl) await updateGitHubAvatar();
     if (!settings.artifactStore) await updateArtifactStore();
     success(res, settings);

backend/src/restful/sync.js

@@ -22,24 +22,60 @@ export default function register($app) {
     $app.get('/api/sync/artifact/:name', syncArtifact);
 }
 
-async function produceArtifact({ type, name, platform, url, ua, content }) {
+async function produceArtifact({
+    type,
+    name,
+    platform,
+    url,
+    ua,
+    content,
+    mergeSources,
+}) {
     platform = platform || 'JSON';
 
     if (type === 'subscription') {
         const allSubs = $.read(SUBS_KEY);
         const sub = findByName(allSubs, name);
         let raw;
-        if (url) {
-            raw = await download(url, ua);
-        } else if (content) {
+        if (content && !['localFirst', 'remoteFirst'].includes(mergeSources)) {
             raw = content;
-        } else if (sub.source === 'local') {
+        } else if (url) {
+            raw = await Promise.all(
+                url
+                    .split(/[\r\n]+/)
+                    .map((i) => i.trim())
+                    .filter((i) => i.length)
+                    .map((url) => download(url, ua)),
+            );
+            if (mergeSources === 'localFirst') {
+                raw.unshift(content);
+            } else if (mergeSources === 'remoteFirst') {
+                raw.push(content);
+            }
+        } else if (
+            sub.source === 'local' &&
+            !['localFirst', 'remoteFirst'].includes(sub.mergeSources)
+        ) {
             raw = sub.content;
         } else {
-            raw = await download(sub.url, sub.ua);
+            raw = await Promise.all(
+                sub.url
+                    .split(/[\r\n]+/)
+                    .map((i) => i.trim())
+                    .filter((i) => i.length)
+                    .map((url) => download(url, sub.ua)),
+            );
+            if (sub.mergeSources === 'localFirst') {
+                raw.unshift(sub.content);
+            } else if (sub.mergeSources === 'remoteFirst') {
+                raw.push(sub.content);
+            }
         }
         // parse proxies
-        let proxies = ProxyUtils.parse(raw);
+        let proxies = (Array.isArray(raw) ? raw : [raw])
+            .map((i) => ProxyUtils.parse(i))
+            .flat();
+
         proxies.forEach((proxy) => {
             proxy.subName = sub.name;
         });
@@ -87,13 +123,31 @@ async function produceArtifact({ type, name, platform, url, ua, content }) {
                 try {
                     $.info(`正在处理子订阅：${sub.name}...`);
                     let raw;
-                    if (sub.source === 'local') {
+                    if (
+                        sub.source === 'local' &&
+                        !['localFirst', 'remoteFirst'].includes(
+                            sub.mergeSources,
+                        )
+                    ) {
                         raw = sub.content;
                     } else {
-                        raw = await download(sub.url, sub.ua);
+                        raw = await await Promise.all(
+                            sub.url
+                                .split(/[\r\n]+/)
+                                .map((i) => i.trim())
+                                .filter((i) => i.length)
+                                .map((url) => download(url, sub.ua)),
+                        );
+                        if (sub.mergeSources === 'localFirst') {
+                            raw.unshift(sub.content);
+                        } else if (sub.mergeSources === 'remoteFirst') {
+                            raw.push(sub.content);
+                        }
                     }
                     // parse proxies
-                    let currentProxies = ProxyUtils.parse(raw);
+                    let currentProxies = (Array.isArray(raw) ? raw : [raw])
+                        .map((i) => ProxyUtils.parse(i))
+                        .flat();
 
                     currentProxies.forEach((proxy) => {
                         proxy.subName = sub.name;

backend/src/utils/download.js

@@ -1,4 +1,4 @@
-import { FILES_KEY, MODULES_KEY } from '@/constants';
+import { FILES_KEY, MODULES_KEY, SETTINGS_KEY } from '@/constants';
 import { findByName } from '@/utils/database';
 import { HTTP, ENV } from '@/vendor/open-api';
 import { hex_md5 } from '@/vendor/md5';
@@ -45,7 +45,8 @@ export default async function download(url, ua) {
     }
 
     const { isNode } = ENV();
-    ua = ua || 'Quantumult%20X/1.0.29 (iPhone14,5; iOS 15.4.1)';
+    const { defaultUserAgent } = $.read(SETTINGS_KEY);
+    ua = ua || defaultUserAgent || 'clash.meta';
     const id = hex_md5(ua + url);
     if (!isNode && tasks.has(id)) {
         return tasks.get(id);
@@ -63,6 +64,7 @@ export default async function download(url, ua) {
         if (!$arguments?.noCache && cached) {
             resolve(cached);
         } else {
+            $.info(`Downloading...\nUser-Agent: ${ua}\nURL: ${url}`);
             http.get(url)
                 .then((resp) => {
                     const body = resp.body;

backend/src/utils/flow.js

@@ -3,7 +3,10 @@ import { HTTP } from '@/vendor/open-api';
 export async function getFlowHeaders(url) {
     const http = HTTP();
     const { headers } = await http.get({
-        url,
+        url: url
+            .split(/[\r\n]+/)
+            .map((i) => i.trim())
+            .filter((i) => i.length)[0],
         headers: {
             'User-Agent': 'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)',
         },

backend/src/vendor/express.js

@@ -1,8 +1,9 @@
 /* eslint-disable no-undef */
 import { ENV } from './open-api';
 
-export default function express({ substore: $, port }) {
+export default function express({ substore: $, port, host }) {
     port = port || 3000;
+    host = host || '::';
     const { isNode } = ENV();
     const DEFAULT_HEADERS = {
         'Content-Type': 'text/plain;charset=UTF-8',
@@ -29,8 +30,9 @@ export default function express({ substore: $, port }) {
 
         // adapter
         app.start = () => {
-            app.listen(port, () => {
-                $.info(`Express started on port: ${port}`);
+            const listener = app.listen(port, host, () => {
+                const { address, port } = listener.address();
+                $.info(`Express started on ${address}:${port}`);
             });
         };
         return app;