refactor: Migrate to API v2

- Added auto schema migration
- Refactored /api/subs, /api/collections, /api/artifacts. Now these APIs will return array instead of object. This enables sorting items in the future.
This commit is contained in:
Peng-YM
2022-07-05 10:59:40 +08:00
parent b1151859b3
commit 84b4dba425
19 changed files with 335 additions and 277 deletions

View File

@@ -11,7 +11,9 @@ import {
COLLECTIONS_KEY,
RULES_KEY,
SETTINGS_KEY,
} from './constants';
} from '@/constants';
import { deleteByName, findByName, updateByName } from '@/utils/database';
import { success } from '@/restful/response';
export default function register($app) {
// Initialization
@@ -31,10 +33,7 @@ export default function register($app) {
function getAllArtifacts(req, res) {
const allArtifacts = $.read(ARTIFACTS_KEY);
res.json({
status: 'success',
data: allArtifacts,
});
success(res, allArtifacts);
}
async function getArtifact(req, res) {
@@ -42,32 +41,25 @@ async function getArtifact(req, res) {
name = decodeURIComponent(name);
const action = req.query.action;
const allArtifacts = $.read(ARTIFACTS_KEY);
const artifact = allArtifacts[name];
const artifact = findByName(allArtifacts, name);
if (artifact) {
if (action) {
let item;
switch (artifact.type) {
case 'subscription':
item = $.read(SUBS_KEY)[artifact.source];
break;
case 'collection':
item = $.read(COLLECTIONS_KEY)[artifact.source];
break;
case 'rule':
item = $.read(RULES_KEY)[artifact.source];
break;
}
const output = await produceArtifact({
type: artifact.type,
item,
name: artifact.source,
platform: artifact.platform,
});
if (action === 'preview') {
res.send(output);
} else if (action === 'sync') {
$.info(`正在上传配置:${artifact.name}\n>>>`);
console.log(JSON.stringify(artifact, null, 2));
$.info(
`正在上传配置:${artifact.name}\n>>>${JSON.stringify(
artifact,
null,
2,
)}`,
);
try {
const resp = await syncArtifact({
[encodeURIComponent(artifact.name)]: {
@@ -80,9 +72,7 @@ async function getArtifact(req, res) {
encodeURIComponent(artifact.name)
].raw_url.replace(/\/raw\/[^/]*\/(.*)/, '/raw/$1');
$.write(allArtifacts, ARTIFACTS_KEY);
res.json({
status: 'success',
});
success(res);
} catch (err) {
res.status(500).json({
status: 'failed',
@@ -91,10 +81,7 @@ async function getArtifact(req, res) {
}
}
} else {
res.json({
status: 'success',
data: artifact,
});
success(res, artifact);
}
} else {
res.status(404).json({
@@ -108,18 +95,15 @@ function createArtifact(req, res) {
const artifact = req.body;
$.info(`正在创建远程配置:${artifact.name}`);
const allArtifacts = $.read(ARTIFACTS_KEY);
if (allArtifacts[artifact.name]) {
if (findByName(allArtifacts, artifact.name)) {
res.status(500).json({
status: 'failed',
message: `远程配置${artifact.name}已存在!`,
});
} else {
allArtifacts[artifact.name] = artifact;
allArtifacts.push(artifact);
$.write(allArtifacts, ARTIFACTS_KEY);
res.status(201).json({
status: 'success',
data: artifact,
});
success(res, artifact, 201);
}
}
@@ -127,31 +111,16 @@ function updateArtifact(req, res) {
const allArtifacts = $.read(ARTIFACTS_KEY);
let oldName = req.params.name;
oldName = decodeURIComponent(oldName);
const artifact = allArtifacts[oldName];
const artifact = findByName(allArtifacts, oldName);
if (artifact) {
$.info(`正在更新远程配置:${artifact.name}`);
const newArtifact = req.body;
if (
typeof newArtifact.name !== 'undefined' &&
!/^[\w-_.]*$/.test(newArtifact.name)
) {
res.status(500).json({
status: 'failed',
message: `远程配置名称 ${newArtifact.name} 中含有非法字符!名称中只能包含英文字母、数字、下划线、横杠。`,
});
} else {
const merged = {
...artifact,
...newArtifact,
};
allArtifacts[merged.name] = merged;
if (merged.name !== oldName) delete allArtifacts[oldName];
$.write(allArtifacts, ARTIFACTS_KEY);
res.json({
status: 'success',
data: merged,
});
}
const newArtifact = {
...artifact,
...req.body,
};
updateByName(allArtifacts, oldName, newArtifact);
$.write(allArtifacts, ARTIFACTS_KEY);
success(res, newArtifact);
} else {
res.status(404).json({
status: 'failed',
@@ -166,7 +135,7 @@ async function deleteArtifact(req, res) {
$.info(`正在删除远程配置:${name}`);
const allArtifacts = $.read(ARTIFACTS_KEY);
try {
const artifact = allArtifacts[name];
const artifact = findByName(allArtifacts, name);
if (!artifact) throw new Error(`远程配置:${name}不存在!`);
if (artifact.updated) {
// delete gist
@@ -177,11 +146,9 @@ async function deleteArtifact(req, res) {
await syncArtifact(files);
}
// delete local cache
delete allArtifacts[name];
deleteByName(allArtifacts, name);
$.write(allArtifacts, ARTIFACTS_KEY);
res.json({
status: 'success',
});
success(res);
} catch (err) {
$.error(`无法删除远程配置:${name},原因:${err}`);
res.status(500).json({
@@ -198,24 +165,12 @@ async function cronSyncArtifacts(_, res) {
try {
await Promise.all(
Object.values(allArtifacts).map(async (artifact) => {
allArtifacts.map(async (artifact) => {
if (artifact.sync) {
$.info(`正在同步云配置:${artifact.name}...`);
let item;
switch (artifact.type) {
case 'subscription':
item = $.read(SUBS_KEY)[artifact.source];
break;
case 'collection':
item = $.read(COLLECTIONS_KEY)[artifact.source];
break;
case 'rule':
item = $.read(RULES_KEY)[artifact.source];
break;
}
const output = await produceArtifact({
type: artifact.type,
item,
name: artifact.source,
platform: artifact.platform,
});
@@ -229,7 +184,7 @@ async function cronSyncArtifacts(_, res) {
const resp = await syncArtifact(files);
const body = JSON.parse(resp.body);
for (const artifact of Object.values(allArtifacts)) {
for (const artifact of allArtifacts) {
artifact.updated = new Date().getTime();
// extract real url from gist
artifact.url = body.files[artifact.name].raw_url.replace(
@@ -240,7 +195,7 @@ async function cronSyncArtifacts(_, res) {
$.write(allArtifacts, ARTIFACTS_KEY);
$.info('全部订阅同步成功!');
res.status(200).end();
success(res);
} catch (err) {
res.status(500).json({
error: err,
@@ -261,12 +216,12 @@ async function syncArtifact(files) {
return manager.upload(files);
}
async function produceArtifact({ type, item, platform, noProcessor }) {
async function produceArtifact({ type, name, platform }) {
platform = platform || 'JSON';
noProcessor = noProcessor || false;
if (type === 'subscription') {
const sub = item;
const allSubs = $.read(SUBS_KEY);
const sub = findByName(allSubs, name);
let raw;
if (sub.source === 'local') {
raw = sub.content;
@@ -275,14 +230,12 @@ async function produceArtifact({ type, item, platform, noProcessor }) {
}
// parse proxies
let proxies = ProxyUtils.parse(raw);
if (!noProcessor) {
// apply processors
proxies = await ProxyUtils.process(
proxies,
sub.process || [],
platform,
);
}
// apply processors
proxies = await ProxyUtils.process(
proxies,
sub.process || [],
platform,
);
// check duplicate
const exist = {};
for (const proxy of proxies) {
@@ -304,14 +257,15 @@ async function produceArtifact({ type, item, platform, noProcessor }) {
return ProxyUtils.produce(proxies, platform);
} else if (type === 'collection') {
const allSubs = $.read(SUBS_KEY);
const collection = item;
const subnames = collection['subscriptions'];
const allCols = $.read(COLLECTIONS_KEY);
const collection = findByName(allCols, name);
const subnames = collection.subscriptions;
const results = {};
let processed = 0;
await Promise.all(
subnames.map(async (name) => {
const sub = allSubs[name];
const sub = findByName(allSubs, name);
try {
$.info(`正在处理子订阅:${sub.name}...`);
let raw;
@@ -322,14 +276,12 @@ async function produceArtifact({ type, item, platform, noProcessor }) {
}
// parse proxies
let currentProxies = ProxyUtils.parse(raw);
if (!noProcessor) {
// apply processors
currentProxies = await ProxyUtils.process(
currentProxies,
sub.process || [],
platform,
);
}
// apply processors
currentProxies = await ProxyUtils.process(
currentProxies,
sub.process || [],
platform,
);
results[name] = currentProxies;
processed++;
$.info(
@@ -356,14 +308,12 @@ async function produceArtifact({ type, item, platform, noProcessor }) {
subnames.map((name) => results[name]),
);
if (!noProcessor) {
// apply own processors
proxies = await ProxyUtils.process(
proxies,
collection.process || [],
platform,
);
}
// apply own processors
proxies = await ProxyUtils.process(
proxies,
collection.process || [],
platform,
);
if (proxies.length === 0) {
throw new Error(`组合订阅中不含有效节点!`);
}
@@ -386,7 +336,8 @@ async function produceArtifact({ type, item, platform, noProcessor }) {
}
return ProxyUtils.produce(proxies, platform);
} else if (type === 'rule') {
const rule = item;
const allRules = $.read(RULES_KEY);
const rule = findByName(allRules, name);
let rules = [];
for (let i = 0; i < rule.urls.length; i++) {
const url = rule.urls[i];

View File

@@ -1,4 +1,6 @@
import { COLLECTIONS_KEY } from './constants';
import { deleteByName, findByName, updateByName } from '@/utils/database';
import { COLLECTIONS_KEY } from '@/constants';
import { success } from '@/restful/response';
import $ from '@/core/app';
export default function register($app) {
@@ -18,30 +20,25 @@ export default function register($app) {
function createCollection(req, res) {
const collection = req.body;
$.info(`正在创建组合订阅:${collection.name}`);
const allCol = $.read(COLLECTIONS_KEY);
if (allCol[collection.name]) {
const allCols = $.read(COLLECTIONS_KEY);
if (findByName(allCols, collection.name)) {
res.status(500).json({
status: 'failed',
message: `订阅集${collection.name}已存在!`,
});
}
allCol[collection.name] = collection;
$.write(allCol, COLLECTIONS_KEY);
res.status(201).json({
status: 'success',
data: collection,
});
allCols.push(collection);
$.write(allCols, COLLECTIONS_KEY);
success(res, collection, 201);
}
function getCollection(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
const collection = $.read(COLLECTIONS_KEY)[name];
const allCols = $.read(COLLECTIONS_KEY);
const collection = findByName(allCols, name);
if (collection) {
res.json({
status: 'success',
data: collection,
});
success(res, collection);
} else {
res.status(404).json({
status: 'failed',
@@ -54,21 +51,17 @@ function updateCollection(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
let collection = req.body;
const allCol = $.read(COLLECTIONS_KEY);
if (allCol[name]) {
const allCols = $.read(COLLECTIONS_KEY);
const oldCol = findByName(allCols, name);
if (oldCol) {
const newCol = {
...allCol[name],
...oldCol,
...collection,
};
$.info(`正在更新组合订阅:${name}...`);
// allow users to update collection name
delete allCol[name];
allCol[collection.name || name] = newCol;
$.write(allCol, COLLECTIONS_KEY);
res.json({
status: 'success',
data: newCol,
});
updateByName(allCols, name, newCol);
$.write(allCols, COLLECTIONS_KEY);
success(res, newCol);
} else {
res.status(500).json({
status: 'failed',
@@ -81,18 +74,13 @@ function deleteCollection(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
$.info(`正在删除组合订阅:${name}`);
let allCol = $.read(COLLECTIONS_KEY);
delete allCol[name];
$.write(allCol, COLLECTIONS_KEY);
res.json({
status: 'success',
});
let allCols = $.read(COLLECTIONS_KEY);
deleteByName(allCols, name);
$.write(allCols, COLLECTIONS_KEY);
success(res);
}
function getAllCollections(req, res) {
const allCols = $.read(COLLECTIONS_KEY);
res.json({
status: 'success',
data: allCols,
});
success(res, allCols);
}

View File

@@ -1,8 +0,0 @@
export const SETTINGS_KEY = 'settings';
export const SUBS_KEY = 'subs';
export const COLLECTIONS_KEY = 'collections';
export const ARTIFACTS_KEY = 'artifacts';
export const RULES_KEY = 'rules';
export const GIST_BACKUP_KEY = 'Auto Generated Sub-Store Backup';
export const GIST_BACKUP_FILE_NAME = 'Sub-Store';
export const ARTIFACT_REPOSITORY_KEY = 'Sub-Store Artifacts Repository';

View File

@@ -1,5 +1,6 @@
import { getPlatformFromHeaders } from '@/utils/platform';
import { COLLECTIONS_KEY, SUBS_KEY } from './constants';
import { COLLECTIONS_KEY, SUBS_KEY } from '@/constants';
import { findByName } from '@/utils/database';
import { getFlowHeaders } from '@/utils/flow';
import { produceArtifact } from './artifacts';
import $ from '@/core/app';
@@ -13,21 +14,19 @@ async function downloadSubscription(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
const raw = req.query.raw || false;
const platform =
req.query.target || getPlatformFromHeaders(req.headers) || 'JSON';
$.info(`正在下载订阅:${name}`);
const allSubs = $.read(SUBS_KEY);
const sub = allSubs[name];
const sub = findByName(allSubs, name);
if (sub) {
try {
const output = await produceArtifact({
type: 'subscription',
item: sub,
name,
platform,
noProcessor: raw,
});
if (sub.source !== 'local') {
@@ -69,36 +68,35 @@ async function downloadCollection(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
const { raw } = req.query || 'false';
const platform =
req.query.target || getPlatformFromHeaders(req.headers) || 'JSON';
const allCollections = $.read(COLLECTIONS_KEY);
const collection = allCollections[name];
const allCols = $.read(COLLECTIONS_KEY);
const collection = findByName(allCols, name);
$.info(`正在下载组合订阅:${name}`);
// forward flow header from the first subscription in this collection
const allSubs = $.read(SUBS_KEY);
const subs = collection['subscriptions'];
if (subs.length > 0) {
const sub = allSubs[subs[0]];
if (sub.source !== 'local') {
const flowInfo = await getFlowHeaders(sub.url);
if (flowInfo) {
res.set('subscription-userinfo', flowInfo);
}
}
}
if (collection) {
try {
const output = await produceArtifact({
type: 'collection',
item: collection,
name,
platform,
noProcessor: raw,
});
// forward flow header from the first subscription in this collection
const allSubs = $.read(SUBS_KEY);
const subnames = collection.subscriptions;
if (subnames.length > 0) {
const sub = findByName(allSubs, subnames[0]);
if (sub.source !== 'local') {
const flowInfo = await getFlowHeaders(sub.url);
if (flowInfo) {
res.set('subscription-userinfo', flowInfo);
}
}
}
if (platform === 'JSON') {
res.set('Content-Type', 'application/json;charset=utf-8').send(
output,

View File

@@ -2,7 +2,7 @@ import {
SETTINGS_KEY,
GIST_BACKUP_KEY,
GIST_BACKUP_FILE_NAME,
} from './constants';
} from '@/constants';
import { version as substoreVersion } from '../../package.json';
import { ENV, HTTP } from '@/vendor/open-api';
import express from '@/vendor/express';
@@ -15,6 +15,7 @@ import registerArtifactRoutes from './artifacts';
import registerDownloadRoutes from './download';
import registerSettingRoutes from './settings';
import registerPreviewRoutes from './preview';
import { success } from '@/restful/response';
export default function serve() {
const $app = express({ substore: $ });
@@ -117,15 +118,12 @@ async function gistBackup(req, res) {
$.write(content, '#sub-store');
if ($.env.isNode) {
content = JSON.parse(content);
Object.keys(content).forEach((key) => {
$.write(content[key], key);
});
$.cache = content;
$.persistCache();
}
break;
}
res.json({
status: 'success',
});
success(res);
} catch (err) {
const msg = `${
action === 'upload' ? '上传' : '下载'

View File

@@ -1,8 +1,9 @@
import { InternalServerError, NetworkError } from './errors';
import { ProxyUtils } from '@/core/proxy-utils';
import { findByName } from '@/utils/database';
import { success, failed } from './response';
import download from '@/utils/download';
import { SUBS_KEY } from './constants';
import { SUBS_KEY } from '@/constants';
import $ from '@/core/app';
export default function register($app) {
@@ -53,12 +54,12 @@ async function compareSub(req, res) {
async function compareCollection(req, res) {
const allSubs = $.read(SUBS_KEY);
const collection = req.body;
const subnames = collection['subscriptions'];
const subnames = collection.subscriptions;
const results = {};
await Promise.all(
subnames.map(async (name) => {
const sub = allSubs[name];
const sub = findByName(allSubs, name);
try {
let raw;
if (sub.source === 'local') {

View File

@@ -1,4 +1,5 @@
import { SETTINGS_KEY } from './constants';
import { SETTINGS_KEY } from '@/constants';
import { success } from './response';
import $ from '@/core/app';
export default function register($app) {
@@ -8,20 +9,15 @@ export default function register($app) {
function getSettings(req, res) {
const settings = $.read(SETTINGS_KEY);
res.json(settings);
success(res, settings);
}
function updateSettings(req, res) {
const data = req.body;
const settings = $.read(SETTINGS_KEY);
$.write(
{
...settings,
...data,
},
SETTINGS_KEY,
);
res.json({
status: 'success',
});
const newSettings = {
...settings,
...req.body,
};
$.write(newSettings, SETTINGS_KEY);
success(res, newSettings);
}

View File

@@ -3,13 +3,15 @@ import {
InternalServerError,
ResourceNotFoundError,
} from './errors';
import { SUBS_KEY, COLLECTIONS_KEY } from './constants';
import { deleteByName, findByName, updateByName } from '@/utils/database';
import { SUBS_KEY, COLLECTIONS_KEY } from '@/constants';
import { getFlowHeaders } from '@/utils/flow';
import { success, failed } from './response';
import $ from '@/core/app';
if (!$.read(SUBS_KEY)) $.write({}, SUBS_KEY);
export default function register($app) {
if (!$.read(SUBS_KEY)) $.write({}, SUBS_KEY);
$app.get('/api/sub/flow/:name', getFlowInfo);
$app.route('/api/sub/:name')
@@ -24,8 +26,8 @@ export default function register($app) {
async function getFlowInfo(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
const sub = $.read(SUBS_KEY)[name];
const allSubs = $.read(SUBS_KEY);
const sub = findByName(allSubs, name);
if (!sub) {
failed(
res,
@@ -71,31 +73,26 @@ async function getFlowInfo(req, res) {
function createSubscription(req, res) {
const sub = req.body;
const allSubs = $.read(SUBS_KEY);
$.info(`正在创建订阅: ${sub.name}`);
if (allSubs[sub.name]) {
const allSubs = $.read(SUBS_KEY);
if (findByName(allSubs, sub.name)) {
res.status(500).json({
status: 'failed',
message: `订阅${sub.name}已存在!`,
});
}
allSubs[sub.name] = sub;
allSubs.push(sub);
$.write(allSubs, SUBS_KEY);
res.status(201).json({
status: 'success',
data: sub,
});
success(res, sub, 201);
}
function getSubscription(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
const sub = $.read(SUBS_KEY)[name];
const allSubs = $.read(SUBS_KEY);
const sub = findByName(allSubs, name);
if (sub) {
res.json({
status: 'success',
data: sub,
});
success(res, sub);
} else {
res.status(404).json({
status: 'failed',
@@ -106,12 +103,13 @@ function getSubscription(req, res) {
function updateSubscription(req, res) {
let { name } = req.params;
name = decodeURIComponent(name);
name = decodeURIComponent(name); // the original name
let sub = req.body;
const allSubs = $.read(SUBS_KEY);
if (allSubs[name]) {
const oldSub = findByName(allSubs, name);
if (oldSub) {
const newSub = {
...allSubs[name],
...oldSub,
...sub,
};
$.info(`正在更新订阅: ${name}`);
@@ -119,23 +117,16 @@ function updateSubscription(req, res) {
if (name !== sub.name) {
// we need to find out all collections refer to this name
const allCols = $.read(COLLECTIONS_KEY);
for (const k of Object.keys(allCols)) {
const idx = allCols[k].subscriptions.indexOf(name);
for (const collection of allCols) {
const idx = collection.subscriptions.indexOf(name);
if (idx !== -1) {
allCols[k].subscriptions[idx] = sub.name;
collection.subscriptions[idx] = sub.name;
}
}
// update subscriptions
delete allSubs[name];
allSubs[sub.name] = newSub;
} else {
allSubs[name] = newSub;
}
updateByName(allSubs, name, newSub);
$.write(allSubs, SUBS_KEY);
res.json({
status: 'success',
data: newSub,
});
success(res, newSub);
} else {
res.status(500).json({
status: 'failed',
@@ -150,25 +141,20 @@ function deleteSubscription(req, res) {
$.info(`删除订阅:${name}...`);
// delete from subscriptions
let allSubs = $.read(SUBS_KEY);
delete allSubs[name];
deleteByName(allSubs, name);
$.write(allSubs, SUBS_KEY);
// delete from collections
let allCols = $.read(COLLECTIONS_KEY);
for (const k of Object.keys(allCols)) {
allCols[k].subscriptions = allCols[k].subscriptions.filter(
const allCols = $.read(COLLECTIONS_KEY);
for (const collection of allCols) {
collection.subscriptions = collection.subscriptions.filter(
(s) => s !== name,
);
}
$.write(allCols, COLLECTIONS_KEY);
res.json({
status: 'success',
});
success(res);
}
function getAllSubscriptions(req, res) {
const allSubs = $.read(SUBS_KEY);
res.json({
status: 'success',
data: allSubs,
});
success(res, allSubs);
}