添加Sub-Store分流支持

This commit is contained in:
Peng-YM 2020-11-26 13:43:54 +08:00
parent e630caf386
commit a6374ac9c3
2 changed files with 2154 additions and 1787 deletions

View File

@ -24,10 +24,16 @@ function service() {
const SETTINGS_KEY = "settings";
const SUBS_KEY = "subs";
const COLLECTIONS_KEY = "collections";
const RULES_KEY = "rules";
const BUILT_IN_KEY = "builtin";
// Initialization
if (!$.read(SUBS_KEY)) $.write({}, SUBS_KEY);
if (!$.read(COLLECTIONS_KEY)) $.write({}, COLLECTIONS_KEY);
if (!$.read(SETTINGS_KEY)) $.write({}, SETTINGS_KEY);
if (!$.read(RULES_KEY)) $.write({}, RULES_KEY);
$.write({
rules: getBuiltInRules(),
}, BUILT_IN_KEY);
// download
$app.get("/download/collection/:name", downloadCollection);
@ -53,6 +59,9 @@ function service() {
.get(getAllCollections)
.post(createCollection);
// rules API
$app.get("/download/rule/:name", downloadRule);
// gist backup
$app.get("/api/backup");
@ -84,10 +93,12 @@ function service() {
res.set("location", "https://sub-store.vercel.app/").status(302).end();
});
// handle preflight request
// handle preflight request for QX
if (ENV().isQX) {
$app.options("/", async (req, res) => {
res.status(200).end();
});
}
$app.all("/", (req, res) => {
res.send("Hello from sub-store, made with ❤️ by Peng-YM");
@ -409,6 +420,48 @@ function service() {
});
}
// rule API
async function downloadRule(req, res) {
const {name} = req.params;
const {builtin} = req.query;
const platform = req.query.target || getPlatformFromHeaders(req.headers);
$.info(`正在下载${builtin ? "内置" : ""}分流订阅:${name}...`);
let rule;
if (builtin) {
rule = $.read(BUILT_IN_KEY)['rules'][name];
}
if (rule) {
let rules = [];
for (let i = 0; i < rule.urls.length; i++) {
const url = rule.urls[i];
$.info(`正在处理URL${url},进度--${100 * ((i + 1) / rule.urls.length).toFixed(1)}% `);
try {
const {body} = await $.http.get(url);
const currentRules = RuleUtils.parse(body);
rules = rules.concat(currentRules);
} catch (err) {
$.error(`处理分流订阅中的URL: ${url}时出现错误:${err}! 该订阅已被跳过。`);
}
}
// remove duplicates
rules = await RuleUtils.process(rules, [{type: "Remove Duplicate"}]);
// produce output
const output = RuleUtils.produce(rules, platform);
res.send(output);
} else {
// rule not found
$.notify(
`🌍 [Sub-Store] 下载分流订阅失败`,
`❌ 未找到分流订阅:${name}`,
);
res.status(404).json({
status: "failed",
});
}
}
// settings API
function getSettings(req, res) {
const settings = $.read(SETTINGS_KEY);
@ -546,165 +599,34 @@ function service() {
$.log(`Use cached for url: ${url}`);
return resource;
}
const {body} = await $http.get(url);
let body = "";
try {
body = await $http.get(url);
} catch (err) {
throw new Error(err);
} finally {
$.write(body, key);
$.write(new Date().getTime(), timeKey);
}
if (body.replace(/\s/g, "").length === 0) {
throw new Error("订阅内容为空!");
}
$.write(body, key);
$.write(new Date().getTime(), timeKey);
return body;
}
}
/****************************************** Proxy Utils **********************************************************/
var ProxyUtils = (function () {
function preprocess(raw) {
for (const processor of PROXY_PREPROCESSORS) {
try {
if (processor.test(raw)) {
$.log(`Pre-processor [${processor.name}] activated`);
return processor.parse(raw);
}
} catch (e) {
$.error(`Parser [${processor.name}] failed\n Reason: ${e}`);
}
}
return raw;
const PROXY_PREPROCESSORS = (function () {
function HTML() {
const name = "HTML";
const test = raw => /^<!DOCTYPE html>/.test(raw);
// simply discard HTML
const parse = _ => "";
return {name, test, parse};
}
function safeMatch(p, line) {
let patternMatched;
try {
patternMatched = p.test(line);
} catch (err) {
patternMatched = false;
}
return patternMatched;
}
function parse(raw) {
raw = preprocess(raw);
// parse
const lines = raw.split("\n");
const proxies = [];
let lastParser;
for (let line of lines) {
line = line.trim();
if (line.length === 0) continue; // skip empty line
let matched = lastParser && safeMatch(lastParser, line);
if (!matched) {
for (const parser of PROXY_PARSERS) {
if (safeMatch(parser, line)) {
lastParser = parser;
matched = true;
$.log(`Proxy parser: ${parser.name} is activated`);
break;
}
}
}
if (!matched) {
$.error(`Failed to find a rule to parse line: \n${line}\n`);
} else {
try {
const proxy = lastParser.parse(line);
if (!proxy) {
$.error(`Parser ${lastParser.name} return nothing for \n${line}\n`);
}
proxies.push(proxy);
} catch (err) {
$.error(
`Failed to parse line: \n ${line}\n Reason: ${err.stack}`
);
}
}
}
return proxies;
}
async function process(proxies, operators = []) {
for (const item of operators) {
// process script
let script;
if (item.type.indexOf("Script") !== -1) {
const {mode, content} = item.args;
if (mode === "link") {
// if this is remote script, download it
script = await $.http
.get(content)
.then((resp) => resp.body)
.catch((err) => {
throw new Error(
`Error when downloading remote script: ${item.args.content}.\n Reason: ${err}`
);
});
} else {
script = content;
}
}
const op = PROXY_PROCESSORS[item.type];
if (!op) {
$.error(`Unknown operator: "${item.type}"`);
continue;
}
try {
$.log(
`Applying "${item.type}" with arguments:\n >>> ${
JSON.stringify(item.args, null, 2) || "None"
}`
);
if (item.type.indexOf('Script') !== -1) {
proxies = PROXY_PROCESSORS.Apply(op(script), proxies);
} else {
proxies = PROXY_PROCESSORS.Apply(op(item.args), proxies);
}
} catch (err) {
$.error(`Failed to apply "${item.type}"!\n REASON: ${err}`);
}
}
return proxies;
}
function produce(proxies, targetPlatform) {
const producer = PROXY_PRODUCERS[targetPlatform];
if (!producer) {
throw new Error(`Target platform: ${targetPlatform} is not supported!`);
}
// filter unsupported proxies
proxies = proxies.filter(proxy => !(proxy.supported && proxy.supported[targetPlatform] === false));
$.log(`Producing proxies for target: ${targetPlatform}`);
if (typeof producer.type === "undefined" || producer.type === 'SINGLE') {
return proxies
.map(proxy => {
try {
return producer.produce(proxy);
} catch (err) {
$.error(
`Cannot produce proxy: ${JSON.stringify(
proxy, null, 2
)}\nReason: ${err}`
);
return "";
}
})
.filter(line => line.length > 0)
.join("\n");
} else if (producer.type === "ALL") {
return producer.produce(proxies);
}
}
return {
parse, process, produce
}
})();
var PROXY_PREPROCESSORS = (function () {
function Base64Encoded() {
const name = "Base64 Pre-processor";
@ -794,11 +716,10 @@ var PROXY_PREPROCESSORS = (function () {
}
return [
Base64Encoded(), Clash(), SSD()
HTML(), Base64Encoded(), Clash(), SSD()
];
})();
var PROXY_PARSERS = (function () {
})();
const PROXY_PARSERS = (function () {
// Parse SS URI format (only supports new SIP002, legacy format is depreciated).
// reference: https://shadowsocks.org/en/spec/SIP002-URI-Scheme.html
function URI_SS() {
@ -1589,9 +1510,8 @@ var PROXY_PARSERS = (function () {
Loon_SS(), Loon_SSR(), Loon_VMess(), Loon_Trojan(), Loon_Http(),
QX_SS(), QX_SSR(), QX_VMess(), QX_Trojan(), QX_Http()
];
})();
var PROXY_PROCESSORS = (function () {
})();
const PROXY_PROCESSORS = (function () {
// force to set some properties (e.g., skip-cert-verify, udp, tfo, etc.)
function SetPropertyOperator({key, value}) {
return {
@ -2169,9 +2089,8 @@ var PROXY_PROCESSORS = (function () {
"Apply": Apply,
};
})();
var PROXY_PRODUCERS = (function () {
})();
const PROXY_PRODUCERS = (function () {
function QX_Producer() {
const targetPlatform = "QX";
const produce = (proxy) => {
@ -2520,8 +2439,456 @@ var PROXY_PRODUCERS = (function () {
"URI": URI_Producer(),
"JSON": JSON_Producer()
}
})();
function preprocess(raw) {
for (const processor of PROXY_PREPROCESSORS) {
try {
if (processor.test(raw)) {
$.log(`Pre-processor [${processor.name}] activated`);
return processor.parse(raw);
}
} catch (e) {
$.error(`Parser [${processor.name}] failed\n Reason: ${e}`);
}
}
return raw;
}
function safeMatch(p, line) {
let patternMatched;
try {
patternMatched = p.test(line);
} catch (err) {
patternMatched = false;
}
return patternMatched;
}
function parse(raw) {
raw = preprocess(raw);
// parse
const lines = raw.split("\n");
const proxies = [];
let lastParser;
for (let line of lines) {
line = line.trim();
if (line.length === 0) continue; // skip empty line
let matched = lastParser && safeMatch(lastParser, line);
if (!matched) {
for (const parser of PROXY_PARSERS) {
if (safeMatch(parser, line)) {
lastParser = parser;
matched = true;
$.log(`Proxy parser: ${parser.name} is activated`);
break;
}
}
}
if (!matched) {
$.error(`Failed to find a rule to parse line: \n${line}\n`);
} else {
try {
const proxy = lastParser.parse(line);
if (!proxy) {
$.error(`Parser ${lastParser.name} return nothing for \n${line}\n`);
}
proxies.push(proxy);
} catch (err) {
$.error(
`Failed to parse line: \n ${line}\n Reason: ${err.stack}`
);
}
}
}
return proxies;
}
async function process(proxies, operators = []) {
for (const item of operators) {
// process script
let script;
if (item.type.indexOf("Script") !== -1) {
const {mode, content} = item.args;
if (mode === "link") {
// if this is remote script, download it
script = await $.http
.get(content)
.then((resp) => resp.body)
.catch((err) => {
throw new Error(
`Error when downloading remote script: ${item.args.content}.\n Reason: ${err}`
);
});
} else {
script = content;
}
}
const op = PROXY_PROCESSORS[item.type];
if (!op) {
$.error(`Unknown operator: "${item.type}"`);
continue;
}
try {
$.log(
`Applying "${item.type}" with arguments:\n >>> ${
JSON.stringify(item.args, null, 2) || "None"
}`
);
if (item.type.indexOf('Script') !== -1) {
proxies = PROXY_PROCESSORS.Apply(op(script), proxies);
} else {
proxies = PROXY_PROCESSORS.Apply(op(item.args), proxies);
}
} catch (err) {
$.error(`Failed to apply "${item.type}"!\n REASON: ${err}`);
}
}
return proxies;
}
function produce(proxies, targetPlatform) {
const producer = PROXY_PRODUCERS[targetPlatform];
if (!producer) {
throw new Error(`Target platform: ${targetPlatform} is not supported!`);
}
// filter unsupported proxies
proxies = proxies.filter(proxy => !(proxy.supported && proxy.supported[targetPlatform] === false));
$.log(`Producing proxies for target: ${targetPlatform}`);
if (typeof producer.type === "undefined" || producer.type === 'SINGLE') {
return proxies
.map(proxy => {
try {
return producer.produce(proxy);
} catch (err) {
$.error(
`Cannot produce proxy: ${JSON.stringify(
proxy, null, 2
)}\nReason: ${err}`
);
return "";
}
})
.filter(line => line.length > 0)
.join("\n");
} else if (producer.type === "ALL") {
return producer.produce(proxies);
}
}
return {
parse, process, produce
}
})();
/****************************************** Rule Utils **********************************************************/
var RuleUtils = (function () {
const RULE_PARSERS = (function () {
// Rule set format for Surge
function SurgeRuleSet() {
const name = "Surge Rule Set Parser"
const SURGE_RULE_TYPES = [
// Domain-based rules
"DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD",
// IP based rules
"IP-CIDR", "IP-CIDR6",
// HTTP rules
"USER-AGENT", "URL-REGEX",
// Misc rules
"DEST-PORT", "SRC-IP", "IN-PORT", "PROTOCOL"
];
const test = (raw) => (
raw.indexOf("payload:") !== 0 &&
SURGE_RULE_TYPES.some(k => raw.indexOf(k) !== -1)
);
const parse = (raw) => {
const lines = raw.split("\n");
const result = [];
for (let line of lines) {
line = line.trim();
// skip comments
if (/\s*#/.test(line)) continue;
if (!SURGE_RULE_TYPES.some(k => line.indexOf(k) === 0)) continue;
try {
const params = line.split(",").map(w => w.trim());
const rule = {
type: params[0],
content: params[1],
};
if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
rule.options = params.slice(2)
}
result.push(rule);
} catch (e) {
console.error(`Failed to parse line: ${line}\n Reason: ${e}`);
}
}
return result;
};
return {name, test, parse};
}
function ClashRuleProvider() {
const name = "Clash Rule Provider";
const CLASH_RULE_TYPES = [
// Domain-based rules
"DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD",
// IP based rules
"IP-CIDR", "IP-CIDR6",
// HTTP rules
"USER-AGENT", "URL-REGEX",
// Process rules
"PROCESS-NAME",
// Misc rules
"DST-PORT", "SRC-IP-CIDR", "SRC-PORT"
];
const test = (raw) => (
raw.indexOf("payload:") === 0 &&
CLASH_RULE_TYPES.some(k => raw.indexOf(k) !== -1)
);
const parse = (raw) => {
const result = [];
try {
const conf = YAML.eval(raw);
const payload = conf["payload"]
.map(
rule => rule.replace("DST-PORT", "DEST-PORT")
.replace("SRC-IP-CIDR", "SRC-IP")
.replace("SRC-PORT", "IN-PORT")
)
.join("\n");
return SurgeRuleSet().parse(payload);
} catch (e) {
console.error(`Cannot parse rules: ${e}`);
}
return result;
};
return {name, test, parse};
}
function QX() {
const name = "QX Filter";
const QX_RULE_TYPES = [
"host", "host-suffix", "host-keyword",
"ip-cidr", "ip6-cidr",
"user-agent"
];
const test = (raw) => (
QX_RULE_TYPES.some(k => raw.indexOf(k.toLowerCase()) === 0)
)
const parse = (raw) => {
const lines = raw.split("\n");
for (let i = 0; i < lines.length; i++) {
lines[i] = lines[i]
.replace(/host-suffix/i, "DOMAIN-SUFFIX")
.replace(/host-keyword/i, "DOMAIN-KEYWORD")
.replace(/host/i, "DOMAIN")
.replace("ip-cidr", "IP-CIDR")
.replace(/ip6-cidr/i, "IP-CIDR6")
.replace("user-agent", "USER-AGENT");
}
return SurgeRuleSet().parse(lines.join("\n"));
};
return {name, test, parse};
}
return [SurgeRuleSet(), QX(), ClashRuleProvider()];
})();
const RULE_PROCESSORS = (function () {
function RemoveDuplicate() {
return {
func: rules => {
const seen = new Set();
const result = [];
rules.forEach(rule => {
const options = rule.options || [];
options.sort();
const key = `${rule.type},${rule.content},${JSON.stringify(options)}`;
if (!seen.has(key)) {
result.push(rule)
seen.add(key);
}
});
return result;
}
}
}
return {
"Remove Duplicate": RemoveDuplicate
};
})();
const RULE_PRODUCERS = (function () {
function QXFilter() {
const type = "SINGLE";
const func = (rule) => {
// skip unsupported rules
const UNSUPPORTED = [
"URL-REGEX", "DEST-PORT", "SRC-IP", "IN-PORT", "PROTOCOL"
];
if (UNSUPPORTED.indexOf(rule.type) !== -1) return null;
const TRANSFORM = {
"DOMAIN-KEYWORD": "HOST-KEYWORD",
"DOMAIN-SUFFIX": "HOST-SUFFIX",
"DOMAIN": "HOST",
"IP-CIDR6": "IP6-CIDR"
};
let output = `${TRANSFORM[rule.type] || rule.type},${rule.content}`;
if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
output += rule.options ? `,${rule.options[0]}` : "";
}
output += ",SUB-STORE"
return output;
}
return {type, func};
}
function SurgeRuleSet() {
const type = "SINGLE";
const func = (rule) => {
let output = `${rule.type},${rule.content}`;
if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
output += rule.options ? `,${rule.options[0]}` : "";
}
return output;
}
return {type, func};
}
function LoonRules() {
const type = "SINGLE";
const func = (rule) => {
// skip unsupported rules
const UNSUPPORTED = [
"DEST-PORT", "SRC-IP", "IN-PORT", "PROTOCOL"
];
if (UNSUPPORTED.indexOf(rule.type) !== -1) return null;
return SurgeRuleSet().func(rule);
}
return {type, func};
}
function ClashRuleProvider() {
const type = "ALL";
const func = (rules) => {
const TRANSFORM = {
"DEST-PORT": "DST-PORT",
"SRC-IP": "SRC-IP-CIDR",
"IN-PORT": "SRC-PORT"
};
const conf = {
payload: rules.map(rule => {
let output = `${TRANSFORM[rule.type] || rule.type},${rule.content}`;
if (rule.type === "IP-CIDR" || rule.type === "IP-CIDR6") {
output += rule.options ? `,${rule.options[0]}` : "";
}
return output;
})
}
return YAML.stringify(conf);
}
return {type, func};
}
return {
"QX": QXFilter(),
"Surge": SurgeRuleSet(),
"Loon": LoonRules(),
"Clash": ClashRuleProvider()
};
})();
function parse(raw) {
for (const parser of RULE_PARSERS) {
let matched;
try {
matched = parser.test(raw);
} catch {
matched = false;
}
if (matched) {
console.log(`Rule parser [${parser.name}] is activated!`);
return parser.parse(raw);
}
}
}
async function process(rules, operators) {
for (const item of operators) {
if (!RULE_PROCESSORS[item.type]) {
console.error(`Unknown operator: ${item.type}!`);
continue;
}
const op = RULE_PROCESSORS[item.type](item.args);
try {
console.log(
`Applying operator "${item.type}" with arguments: \n >>> ${
JSON.stringify(item.args) || "None"
}`
);
rules = op.func(rules);
} catch (err) {
console.error(`Failed to apply operator "${item.type}"!\n REASON: ${err}`);
}
}
return rules;
}
function produce(rules, targetPlatform) {
const producer = RULE_PRODUCERS[targetPlatform];
if (!producer) {
throw new Error(`Target platform: ${targetPlatform} is not supported!`);
}
if (typeof producer.type === "undefined" || producer.type === 'SINGLE') {
return rules
.map(rule => {
try {
return producer.func(rule);
} catch (err) {
console.log(
`ERROR: cannot produce rule: ${JSON.stringify(
rule
)}\nReason: ${err}`
);
return "";
}
})
.filter(line => line.length > 0)
.join("\n");
} else if (producer.type === "ALL") {
return producer.func(rules);
}
}
return {parse, process, produce};
})();
function getBuiltInRules() {
return {
"AD": {
"name": "AD",
"description": "",
"urls": [
"https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-surge.txt",
"https://raw.githubusercontent.com/ACL4SSR/ACL4SSR/master/Clash/Providers/BanAD.yaml"
]
}
};
}
/****************************************** Supporting Functions **********************************************************/
/**
* OpenAPI

File diff suppressed because one or more lines are too long