添加Sub-Store分流支持

backend/sub-store.js

@@ -24,10 +24,16 @@ function service() {
     const SETTINGS_KEY = "settings";
     const SUBS_KEY = "subs";
     const COLLECTIONS_KEY = "collections";
+    const RULES_KEY = "rules";
+    const BUILT_IN_KEY = "builtin";
     // Initialization
     if (!$.read(SUBS_KEY)) $.write({}, SUBS_KEY);
     if (!$.read(COLLECTIONS_KEY)) $.write({}, COLLECTIONS_KEY);
     if (!$.read(SETTINGS_KEY)) $.write({}, SETTINGS_KEY);
+    if (!$.read(RULES_KEY)) $.write({}, RULES_KEY);
+    $.write({
+        rules: getBuiltInRules(),
+    }, BUILT_IN_KEY);
 
     // download
     $app.get("/download/collection/:name", downloadCollection);
@@ -53,6 +59,9 @@ function service() {
         .get(getAllCollections)
         .post(createCollection);
 
+    // rules API
+    $app.get("/download/rule/:name", downloadRule);
+
     // gist backup
     $app.get("/api/backup");
 
@@ -84,10 +93,12 @@ function service() {
         res.set("location", "https://sub-store.vercel.app/").status(302).end();
     });
 
-    // handle preflight request
+    // handle preflight request for QX
+    if (ENV().isQX) {
         $app.options("/", async (req, res) => {
             res.status(200).end();
         });
+    }
 
     $app.all("/", (req, res) => {
         res.send("Hello from sub-store, made with ❤️ by Peng-YM");
@@ -409,6 +420,48 @@ function service() {
         });
     }
 
+    // rule API
+    async function downloadRule(req, res) {
+        const {name} = req.params;
+        const {builtin} = req.query;
+        const platform = req.query.target || getPlatformFromHeaders(req.headers);
+
+        $.info(`正在下载${builtin ? "内置" : ""}分流订阅：${name}...`);
+
+        let rule;
+        if (builtin) {
+            rule = $.read(BUILT_IN_KEY)['rules'][name];
+        }
+        if (rule) {
+            let rules = [];
+            for (let i = 0; i < rule.urls.length; i++) {
+                const url = rule.urls[i];
+                $.info(`正在处理URL：${url}，进度--${100 * ((i + 1) / rule.urls.length).toFixed(1)}% `);
+                try {
+                    const {body} = await $.http.get(url);
+                    const currentRules = RuleUtils.parse(body);
+                    rules = rules.concat(currentRules);
+                } catch (err) {
+                    $.error(`处理分流订阅中的URL: ${url}时出现错误：${err}! 该订阅已被跳过。`);
+                }
+            }
+            // remove duplicates
+            rules = await RuleUtils.process(rules, [{type: "Remove Duplicate"}]);
+            // produce output
+            const output = RuleUtils.produce(rules, platform);
+            res.send(output);
+        } else {
+            // rule not found
+            $.notify(
+                `🌍 [Sub-Store] 下载分流订阅失败`,
+                `❌ 未找到分流订阅：${name}！`,
+            );
+            res.status(404).json({
+                status: "failed",
+            });
+        }
+    }
+
     // settings API
     function getSettings(req, res) {
         const settings = $.read(SETTINGS_KEY);
@@ -546,165 +599,34 @@ function service() {
             $.log(`Use cached for url: ${url}`);
             return resource;
         }
-        const {body} = await $http.get(url);
+
+        let body = "";
+        try {
+            body = await $http.get(url);
+        } catch (err) {
+            throw new Error(err);
+        } finally {
+            $.write(body, key);
+            $.write(new Date().getTime(), timeKey);
+        }
         if (body.replace(/\s/g, "").length === 0) {
             throw new Error("订阅内容为空！");
         }
-        $.write(body, key);
-        $.write(new Date().getTime(), timeKey);
         return body;
     }
 }
 
 /****************************************** Proxy Utils **********************************************************/
 var ProxyUtils = (function () {
-    function preprocess(raw) {
+    const PROXY_PREPROCESSORS = (function () {
-        for (const processor of PROXY_PREPROCESSORS) {
+        function HTML() {
-            try {
+            const name = "HTML";
-                if (processor.test(raw)) {
+            const test = raw => /^<!DOCTYPE html>/.test(raw);
-                    $.log(`Pre-processor [${processor.name}] activated`);
+            // simply discard HTML
-                    return processor.parse(raw);
+            const parse = _ => "";
-                }
+            return {name, test, parse};
-            } catch (e) {
-                $.error(`Parser [${processor.name}] failed\n Reason: ${e}`);
-            }
-        }
-        return raw;
         }
 
-    function safeMatch(p, line) {
-        let patternMatched;
-        try {
-            patternMatched = p.test(line);
-        } catch (err) {
-            patternMatched = false;
-        }
-        return patternMatched;
-    }
-
-    function parse(raw) {
-        raw = preprocess(raw);
-        // parse
-        const lines = raw.split("\n");
-        const proxies = [];
-        let lastParser;
-
-        for (let line of lines) {
-            line = line.trim();
-            if (line.length === 0) continue; // skip empty line
-            let matched = lastParser && safeMatch(lastParser, line);
-            if (!matched) {
-                for (const parser of PROXY_PARSERS) {
-                    if (safeMatch(parser, line)) {
-                        lastParser = parser;
-                        matched = true;
-                        $.log(`Proxy parser: ${parser.name} is activated`);
-                        break;
-                    }
-                }
-            }
-            if (!matched) {
-                $.error(`Failed to find a rule to parse line: \n${line}\n`);
-            } else {
-                try {
-                    const proxy = lastParser.parse(line);
-                    if (!proxy) {
-                        $.error(`Parser ${lastParser.name} return nothing for \n${line}\n`);
-                    }
-                    proxies.push(proxy);
-                } catch (err) {
-                    $.error(
-                        `Failed to parse line: \n ${line}\n Reason: ${err.stack}`
-                    );
-                }
-            }
-        }
-
-        return proxies;
-    }
-
-    async function process(proxies, operators = []) {
-        for (const item of operators) {
-            // process script
-            let script;
-            if (item.type.indexOf("Script") !== -1) {
-                const {mode, content} = item.args;
-                if (mode === "link") {
-                    // if this is remote script, download it
-                    script = await $.http
-                        .get(content)
-                        .then((resp) => resp.body)
-                        .catch((err) => {
-                            throw new Error(
-                                `Error when downloading remote script: ${item.args.content}.\n Reason: ${err}`
-                            );
-                        });
-                } else {
-                    script = content;
-                }
-            }
-
-            const op = PROXY_PROCESSORS[item.type];
-            if (!op) {
-                $.error(`Unknown operator: "${item.type}"`);
-                continue;
-            }
-
-            try {
-                $.log(
-                    `Applying "${item.type}" with arguments:\n >>> ${
-                        JSON.stringify(item.args, null, 2) || "None"
-                    }`
-                );
-                if (item.type.indexOf('Script') !== -1) {
-                    proxies = PROXY_PROCESSORS.Apply(op(script), proxies);
-                } else {
-                    proxies = PROXY_PROCESSORS.Apply(op(item.args), proxies);
-                }
-            } catch (err) {
-                $.error(`Failed to apply "${item.type}"!\n REASON: ${err}`);
-            }
-        }
-        return proxies;
-    }
-
-    function produce(proxies, targetPlatform) {
-        const producer = PROXY_PRODUCERS[targetPlatform];
-        if (!producer) {
-            throw new Error(`Target platform: ${targetPlatform} is not supported!`);
-        }
-
-        // filter unsupported proxies
-        proxies = proxies.filter(proxy => !(proxy.supported && proxy.supported[targetPlatform] === false));
-
-        $.log(`Producing proxies for target: ${targetPlatform}`);
-        if (typeof producer.type === "undefined" || producer.type === 'SINGLE') {
-            return proxies
-                .map(proxy => {
-                    try {
-                        return producer.produce(proxy);
-                    } catch (err) {
-                        $.error(
-                            `Cannot produce proxy: ${JSON.stringify(
-                                proxy, null, 2
-                            )}\nReason: ${err}`
-                        );
-                        return "";
-                    }
-                })
-                .filter(line => line.length > 0)
-                .join("\n");
-        } else if (producer.type === "ALL") {
-            return producer.produce(proxies);
-        }
-    }
-
-    return {
-        parse, process, produce
-    }
-})();
-
-var PROXY_PREPROCESSORS = (function () {
         function Base64Encoded() {
             const name = "Base64 Pre-processor";
 
@@ -794,11 +716,10 @@ var PROXY_PREPROCESSORS = (function () {
         }
 
         return [
-        Base64Encoded(), Clash(), SSD()
+            HTML(), Base64Encoded(), Clash(), SSD()
         ];
-})();
+    })();
-
+    const PROXY_PARSERS = (function () {
-var PROXY_PARSERS = (function () {
         // Parse SS URI format (only supports new SIP002, legacy format is depreciated).
         // reference: https://shadowsocks.org/en/spec/SIP002-URI-Scheme.html
         function URI_SS() {
@@ -1589,9 +1510,8 @@ var PROXY_PARSERS = (function () {
             Loon_SS(), Loon_SSR(), Loon_VMess(), Loon_Trojan(), Loon_Http(),
             QX_SS(), QX_SSR(), QX_VMess(), QX_Trojan(), QX_Http()
         ];
-})();
+    })();
-
+    const PROXY_PROCESSORS = (function () {
-var PROXY_PROCESSORS = (function () {
         // force to set some properties (e.g., skip-cert-verify, udp, tfo, etc.)
         function SetPropertyOperator({key, value}) {
             return {
@@ -2169,9 +2089,8 @@ var PROXY_PROCESSORS = (function () {
 
             "Apply": Apply,
         };
-})();
+    })();
-
+    const PROXY_PRODUCERS = (function () {
-var PROXY_PRODUCERS = (function () {
         function QX_Producer() {
             const targetPlatform = "QX";
             const produce = (proxy) => {
@@ -2520,8 +2439,456 @@ var PROXY_PRODUCERS = (function () {
             "URI": URI_Producer(),
             "JSON": JSON_Producer()
         }
+    })();
+
+    function preprocess(raw) {
+        for (const processor of PROXY_PREPROCESSORS) {
+            try {
+                if (processor.test(raw)) {
+                    $.log(`Pre-processor [${processor.name}] activated`);
+                    return processor.parse(raw);
+                }
+            } catch (e) {
+                $.error(`Parser [${processor.name}] failed\n Reason: ${e}`);
+            }
+        }
+        return raw;
+    }
+
+    function safeMatch(p, line) {
+        let patternMatched;
+        try {
+            patternMatched = p.test(line);
+        } catch (err) {
+            patternMatched = false;
+        }
+        return patternMatched;
+    }
+
+    function parse(raw) {
+        raw = preprocess(raw);
+        // parse
+        const lines = raw.split("\n");
+        const proxies = [];
+        let lastParser;
+
+        for (let line of lines) {
+            line = line.trim();
+            if (line.length === 0) continue; // skip empty line
+            let matched = lastParser && safeMatch(lastParser, line);
+            if (!matched) {
+                for (const parser of PROXY_PARSERS) {
+                    if (safeMatch(parser, line)) {
+                        lastParser = parser;
+                        matched = true;
+                        $.log(`Proxy parser: ${parser.name} is activated`);
+                        break;
+                    }
+                }
+            }
+            if (!matched) {
+                $.error(`Failed to find a rule to parse line: \n${line}\n`);
+            } else {
+                try {
+                    const proxy = lastParser.parse(line);
+                    if (!proxy) {
+                        $.error(`Parser ${lastParser.name} return nothing for \n${line}\n`);
+                    }
+                    proxies.push(proxy);
+                } catch (err) {
+                    $.error(
+                        `Failed to parse line: \n ${line}\n Reason: ${err.stack}`
+                    );
+                }
+            }
+        }
+
+        return proxies;
+    }
+
+    async function process(proxies, operators = []) {
+        for (const item of operators) {
+            // process script
+            let script;
+            if (item.type.indexOf("Script") !== -1) {
+                const {mode, content} = item.args;
+                if (mode === "link") {
+                    // if this is remote script, download it
+                    script = await $.http
+                        .get(content)
+                        .then((resp) => resp.body)
+                        .catch((err) => {
+                            throw new Error(
+                                `Error when downloading remote script: ${item.args.content}.\n Reason: ${err}`
+                            );
+                        });
+                } else {
+                    script = content;
+                }
+            }
+
+            const op = PROXY_PROCESSORS[item.type];
+            if (!op) {
+                $.error(`Unknown operator: "${item.type}"`);
+                continue;
+            }
+
+            try {
+                $.log(
+                    `Applying "${item.type}" with arguments:\n >>> ${
+                        JSON.stringify(item.args, null, 2) || "None"
+                    }`
+                );
+                if (item.type.indexOf('Script') !== -1) {
+                    proxies = PROXY_PROCESSORS.Apply(op(script), proxies);
+                } else {
+                    proxies = PROXY_PROCESSORS.Apply(op(item.args), proxies);
+                }
+            } catch (err) {
+                $.error(`Failed to apply "${item.type}"!\n REASON: ${err}`);
+            }
+        }
+        return proxies;
+    }
+
+    function produce(proxies, targetPlatform) {
+        const producer = PROXY_PRODUCERS[targetPlatform];
+        if (!producer) {
+            throw new Error(`Target platform: ${targetPlatform} is not supported!`);
+        }
+
+        // filter unsupported proxies
+        proxies = proxies.filter(proxy => !(proxy.supported && proxy.supported[targetPlatform] === false));
+
+        $.log(`Producing proxies for target: ${targetPlatform}`);
+        if (typeof producer.type === "undefined" || producer.type === 'SINGLE') {
+            return proxies
+                .map(proxy => {
+                    try {
+                        return producer.produce(proxy);
+                    } catch (err) {
+                        $.error(
+                            `Cannot produce proxy: ${JSON.stringify(
+                                proxy, null, 2
+                            )}\nReason: ${err}`
+                        );
+                        return "";
+                    }
+                })
+                .filter(line => line.length > 0)
+                .join("\n");
+        } else if (producer.type === "ALL") {
+            return producer.produce(proxies);
+        }
+    }
+
+    return {
+        parse, process, produce
+    }
 })();
 
+/****************************************** Rule Utils **********************************************************/
+var RuleUtils = (function () {
+    const RULE_PARSERS = (function () {
+        // Rule set format for Surge
+        function SurgeRuleSet() {
+            const name = "Surge Rule Set Parser"
+
+            const SURGE_RULE_TYPES = [
+                // Domain-based rules
+                "DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD",
+                // IP based rules
+                "IP-CIDR", "IP-CIDR6",
+                // HTTP rules
+                "USER-AGENT", "URL-REGEX",
+                // Misc rules
+                "DEST-PORT", "SRC-IP", "IN-PORT", "PROTOCOL"
+            ];
+
+            const test = (raw) => (
+                raw.indexOf("payload:") !== 0 &&
+                SURGE_RULE_TYPES.some(k => raw.indexOf(k) !== -1)
+            );
+
+            const parse = (raw) => {
+                const lines = raw.split("\n");
+                const result = [];
+                for (let line of lines) {
+                    line = line.trim();
+                    // skip comments
+                    if (/\s*#/.test(line)) continue;
+                    if (!SURGE_RULE_TYPES.some(k => line.indexOf(k) === 0)) continue;
+                    try {
+                        const params = line.split(",").map(w => w.trim());
+                        const rule = {
+                            type: params[0],
+                            content: params[1],
+                        };
+                        if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
+                            rule.options = params.slice(2)
+                        }
+                        result.push(rule);
+                    } catch (e) {
+                        console.error(`Failed to parse line: ${line}\n Reason: ${e}`);
+                    }
+                }
+                return result;
+            };
+
+            return {name, test, parse};
+        }
+
+        function ClashRuleProvider() {
+            const name = "Clash Rule Provider";
+            const CLASH_RULE_TYPES = [
+                // Domain-based rules
+                "DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD",
+                // IP based rules
+                "IP-CIDR", "IP-CIDR6",
+                // HTTP rules
+                "USER-AGENT", "URL-REGEX",
+                // Process rules
+                "PROCESS-NAME",
+                // Misc rules
+                "DST-PORT", "SRC-IP-CIDR", "SRC-PORT"
+            ];
+
+            const test = (raw) => (
+                raw.indexOf("payload:") === 0 &&
+                CLASH_RULE_TYPES.some(k => raw.indexOf(k) !== -1)
+            );
+            const parse = (raw) => {
+                const result = [];
+                try {
+                    const conf = YAML.eval(raw);
+                    const payload = conf["payload"]
+                        .map(
+                            rule => rule.replace("DST-PORT", "DEST-PORT")
+                                .replace("SRC-IP-CIDR", "SRC-IP")
+                                .replace("SRC-PORT", "IN-PORT")
+                        )
+                        .join("\n");
+                    return SurgeRuleSet().parse(payload);
+                } catch (e) {
+                    console.error(`Cannot parse rules: ${e}`);
+                }
+                return result;
+            };
+            return {name, test, parse};
+        }
+
+        function QX() {
+            const name = "QX Filter";
+            const QX_RULE_TYPES = [
+                "host", "host-suffix", "host-keyword",
+                "ip-cidr", "ip6-cidr",
+                "user-agent"
+            ];
+            const test = (raw) => (
+                QX_RULE_TYPES.some(k => raw.indexOf(k.toLowerCase()) === 0)
+            )
+            const parse = (raw) => {
+                const lines = raw.split("\n");
+                for (let i = 0; i < lines.length; i++) {
+                    lines[i] = lines[i]
+                        .replace(/host-suffix/i, "DOMAIN-SUFFIX")
+                        .replace(/host-keyword/i, "DOMAIN-KEYWORD")
+                        .replace(/host/i, "DOMAIN")
+                        .replace("ip-cidr", "IP-CIDR")
+                        .replace(/ip6-cidr/i, "IP-CIDR6")
+                        .replace("user-agent", "USER-AGENT");
+                }
+                return SurgeRuleSet().parse(lines.join("\n"));
+            };
+            return {name, test, parse};
+        }
+
+        return [SurgeRuleSet(), QX(), ClashRuleProvider()];
+    })();
+    const RULE_PROCESSORS = (function () {
+        function RemoveDuplicate() {
+            return {
+                func: rules => {
+                    const seen = new Set();
+                    const result = [];
+                    rules.forEach(rule => {
+                        const options = rule.options || [];
+                        options.sort();
+                        const key = `${rule.type},${rule.content},${JSON.stringify(options)}`;
+                        if (!seen.has(key)) {
+                            result.push(rule)
+                            seen.add(key);
+                        }
+                    });
+                    return result;
+                }
+            }
+        }
+
+        return {
+            "Remove Duplicate": RemoveDuplicate
+        };
+    })();
+    const RULE_PRODUCERS = (function () {
+        function QXFilter() {
+            const type = "SINGLE";
+            const func = (rule) => {
+                // skip unsupported rules
+                const UNSUPPORTED = [
+                    "URL-REGEX", "DEST-PORT", "SRC-IP", "IN-PORT", "PROTOCOL"
+                ];
+                if (UNSUPPORTED.indexOf(rule.type) !== -1) return null;
+
+                const TRANSFORM = {
+                    "DOMAIN-KEYWORD": "HOST-KEYWORD",
+                    "DOMAIN-SUFFIX": "HOST-SUFFIX",
+                    "DOMAIN": "HOST",
+                    "IP-CIDR6": "IP6-CIDR"
+                };
+
+                let output = `${TRANSFORM[rule.type] || rule.type},${rule.content}`;
+                if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
+                    output += rule.options ? `,${rule.options[0]}` : "";
+                }
+                output += ",SUB-STORE"
+                return output;
+            }
+            return {type, func};
+        }
+
+        function SurgeRuleSet() {
+            const type = "SINGLE";
+            const func = (rule) => {
+                let output = `${rule.type},${rule.content}`;
+                if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
+                    output += rule.options ? `,${rule.options[0]}` : "";
+                }
+                return output;
+            }
+            return {type, func};
+        }
+
+        function LoonRules() {
+            const type = "SINGLE";
+            const func = (rule) => {
+                // skip unsupported rules
+                const UNSUPPORTED = [
+                    "DEST-PORT", "SRC-IP", "IN-PORT", "PROTOCOL"
+                ];
+                if (UNSUPPORTED.indexOf(rule.type) !== -1) return null;
+                return SurgeRuleSet().func(rule);
+            }
+            return {type, func};
+        }
+
+        function ClashRuleProvider() {
+            const type = "ALL";
+            const func = (rules) => {
+                const TRANSFORM = {
+                    "DEST-PORT": "DST-PORT",
+                    "SRC-IP": "SRC-IP-CIDR",
+                    "IN-PORT": "SRC-PORT"
+                };
+                const conf = {
+                    payload: rules.map(rule => {
+                        let output = `${TRANSFORM[rule.type] || rule.type},${rule.content}`;
+                        if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
+                            output += rule.options ? `,${rule.options[0]}` : "";
+                        }
+                        return output;
+                    })
+                }
+                return YAML.stringify(conf);
+            }
+            return {type, func};
+        }
+
+        return {
+            "QX": QXFilter(),
+            "Surge": SurgeRuleSet(),
+            "Loon": LoonRules(),
+            "Clash": ClashRuleProvider()
+        };
+    })();
+
+    function parse(raw) {
+        for (const parser of RULE_PARSERS) {
+            let matched;
+            try {
+                matched = parser.test(raw);
+            } catch {
+                matched = false;
+            }
+            if (matched) {
+                console.log(`Rule parser [${parser.name}] is activated!`);
+                return parser.parse(raw);
+            }
+        }
+    }
+
+    async function process(rules, operators) {
+        for (const item of operators) {
+            if (!RULE_PROCESSORS[item.type]) {
+                console.error(`Unknown operator: ${item.type}!`);
+                continue;
+            }
+            const op = RULE_PROCESSORS[item.type](item.args);
+            try {
+                console.log(
+                    `Applying operator "${item.type}" with arguments: \n >>> ${
+                        JSON.stringify(item.args) || "None"
+                    }`
+                );
+                rules = op.func(rules);
+            } catch (err) {
+                console.error(`Failed to apply operator "${item.type}"!\n REASON: ${err}`);
+            }
+        }
+        return rules;
+    }
+
+    function produce(rules, targetPlatform) {
+        const producer = RULE_PRODUCERS[targetPlatform];
+        if (!producer) {
+            throw new Error(`Target platform: ${targetPlatform} is not supported!`);
+        }
+        if (typeof producer.type === "undefined" || producer.type === 'SINGLE') {
+            return rules
+                .map(rule => {
+                    try {
+                        return producer.func(rule);
+                    } catch (err) {
+                        console.log(
+                            `ERROR: cannot produce rule: ${JSON.stringify(
+                                rule
+                            )}\nReason: ${err}`
+                        );
+                        return "";
+                    }
+                })
+                .filter(line => line.length > 0)
+                .join("\n");
+        } else if (producer.type === "ALL") {
+            return producer.func(rules);
+        }
+    }
+
+    return {parse, process, produce};
+})();
+
+function getBuiltInRules() {
+    return {
+        "AD": {
+            "name": "AD",
+            "description": "",
+            "urls": [
+                "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-surge.txt",
+                "https://raw.githubusercontent.com/ACL4SSR/ACL4SSR/master/Clash/Providers/BanAD.yaml"
+            ]
+        }
+    };
+}
+
 /****************************************** Supporting Functions **********************************************************/
 /**
  * OpenAPI