feat: 同步配置前, 预处理订阅, 防止同时请求过多

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.14.216",
+  "version": "2.14.218",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
   "main": "src/main.js",
   "scripts": {

backend/src/core/proxy-utils/producers/surge.js

@@ -14,7 +14,6 @@ const ipVersions = {
 
 export default function Surge_Producer() {
     const produce = (proxy, type, opts = {}) => {
-        console.log(opts);
         switch (proxy.type) {
             case 'ss':
                 return shadowsocks(proxy);

backend/src/products/cron-sync-artifacts.js

@@ -1,8 +1,14 @@
 import { version } from '../../package.json';
-import { SETTINGS_KEY, ARTIFACTS_KEY } from '@/constants';
+import {
+    SETTINGS_KEY,
+    ARTIFACTS_KEY,
+    SUBS_KEY,
+    COLLECTIONS_KEY,
+} from '@/constants';
 import $ from '@/core/app';
 import { produceArtifact } from '@/restful/sync';
 import { syncToGist } from '@/restful/artifacts';
+import { findByName } from '@/utils/database';
 
 !(async function () {
     const settings = $.read(SETTINGS_KEY);
@@ -30,23 +36,83 @@ async function doSync() {
     const files = {};
 
     try {
+        const invalid = [];
+        const allSubs = $.read(SUBS_KEY);
+        const allCols = $.read(COLLECTIONS_KEY);
+        const subNames = [];
+        allArtifacts.map((artifact) => {
+            if (artifact.sync && artifact.source) {
+                if (artifact.type === 'subscription') {
+                    const subName = artifact.source;
+                    const sub = findByName(allSubs, subName);
+                    if (sub && sub.url && !subNames.includes(subName)) {
+                        subNames.push(subName);
+                    }
+                } else if (artifact.type === 'collection') {
+                    const collection = findByName(allCols, artifact.source);
+                    if (collection && Array.isArray(collection.subscriptions)) {
+                        collection.subscriptions.map((subName) => {
+                            const sub = findByName(allSubs, subName);
+                            if (sub && sub.url && !subNames.includes(subName)) {
+                                subNames.push(subName);
+                            }
+                        });
+                    }
+                }
+            }
+        });
+
+        if (subNames.length > 0) {
+            await Promise.all(
+                subNames.map(async (subName) => {
+                    try {
+                        await produceArtifact({
+                            type: 'subscription',
+                            name: subName,
+                        });
+                    } catch (e) {
+                        // $.error(`${e.message ?? e}`);
+                    }
+                }),
+            );
+        }
         await Promise.all(
             allArtifacts.map(async (artifact) => {
-                if (artifact.sync) {
-                    $.info(`正在同步云配置：${artifact.name}...`);
-                    const output = await produceArtifact({
-                        type: artifact.type,
-                        name: artifact.source,
-                        platform: artifact.platform,
-                    });
+                try {
+                    if (artifact.sync && artifact.source) {
+                        $.info(`正在同步云配置：${artifact.name}...`);
+                        const output = await produceArtifact({
+                            type: artifact.type,
+                            name: artifact.source,
+                            platform: artifact.platform,
+                            produceOpts: {
+                                'include-unsupported-proxy':
+                                    artifact.includeUnsupportedProxy,
+                            },
+                        });
 
-                    files[encodeURIComponent(artifact.name)] = {
-                        content: output,
-                    };
+                        // if (!output || output.length === 0)
+                        //     throw new Error('该配置的结果为空 不进行上传');
+
+                        files[encodeURIComponent(artifact.name)] = {
+                            content: output,
+                        };
+                    }
+                } catch (e) {
+                    $.error(
+                        `同步配置 ${artifact.name} 发生错误: ${e.message ?? e}`,
+                    );
+                    invalid.push(artifact.name);
                 }
             }),
         );
 
+        if (invalid.length > 0) {
+            throw new Error(
+                `同步配置 ${invalid.join(', ')} 发生错误 详情请查看日志`,
+            );
+        }
+
         const resp = await syncToGist(files);
         const body = JSON.parse(resp.body);
 
@@ -71,8 +137,8 @@ async function doSync() {
 
         $.write(allArtifacts, ARTIFACTS_KEY);
         $.notify('🌍 Sub-Store', '全部订阅同步成功！');
-    } catch (err) {
-        $.notify('🌍 Sub-Store', '同步订阅失败', `原因：${err}`);
-        $.error(`无法同步订阅配置到 Gist，原因：${err}`);
+    } catch (e) {
+        $.notify('🌍 Sub-Store', '同步订阅失败', `原因：${e.message ?? e}`);
+        $.error(`无法同步订阅配置到 Gist，原因：${e}`);
     }
 }

backend/src/restful/sync.js

@@ -448,6 +448,46 @@ async function syncArtifacts() {
 
     try {
         const invalid = [];
+        const allSubs = $.read(SUBS_KEY);
+        const allCols = $.read(COLLECTIONS_KEY);
+        const subNames = [];
+        allArtifacts.map((artifact) => {
+            if (artifact.sync && artifact.source) {
+                if (artifact.type === 'subscription') {
+                    const subName = artifact.source;
+                    const sub = findByName(allSubs, subName);
+                    if (sub && sub.url && !subNames.includes(subName)) {
+                        subNames.push(subName);
+                    }
+                } else if (artifact.type === 'collection') {
+                    const collection = findByName(allCols, artifact.source);
+                    if (collection && Array.isArray(collection.subscriptions)) {
+                        collection.subscriptions.map((subName) => {
+                            const sub = findByName(allSubs, subName);
+                            if (sub && sub.url && !subNames.includes(subName)) {
+                                subNames.push(subName);
+                            }
+                        });
+                    }
+                }
+            }
+        });
+
+        if (subNames.length > 0) {
+            await Promise.all(
+                subNames.map(async (subName) => {
+                    try {
+                        await produceArtifact({
+                            type: 'subscription',
+                            name: subName,
+                        });
+                    } catch (e) {
+                        // $.error(`${e.message ?? e}`);
+                    }
+                }),
+            );
+        }
+
         await Promise.all(
             allArtifacts.map(async (artifact) => {
                 try {

backend/src/utils/download.js

@@ -74,6 +74,7 @@ export default async function download(rawUrl, ua, timeout) {
     // try to find in app cache
     const cached = resourceCache.get(id);
     if (!$arguments?.noCache && cached) {
+        $.info(`使用缓存: ${url}`);
         result = cached;
     } else {
         $.info(

scripts/demo.js

@@ -35,6 +35,49 @@ function operator(proxies = [], targetPlatform, context) {
   //     yaml, // yaml 解析和生成
   // }
 
+  // 1. Surge 输出 WireGuard 完整配置
+
+  // let proxies = await produceArtifact({
+  //   type: 'subscription',
+  //   name: 'sub',
+  //   platform: 'Surge',
+  //   produceOpts: {
+  //     'include-unsupported-proxy': true,
+  //   }
+  // })
+  // $content = proxies
+  
+  // 2. sing-box
+
+  // 但是一般不需要这样用, 可参考 1. https://t.me/zhetengsha/1111 和 2. https://t.me/zhetengsha/1070
+
+  // let singboxProxies = await produceArtifact({
+  //     type: 'subscription', // type: 'subscription' 或 'collection'
+  //     name: 'sub', // subscription name
+  //     platform: 'sing-box', // target platform
+  //     produceType: 'internal' // 'internal' produces an Array, otherwise produces a String( JSON.parse('JSON String') )
+  // })
+
+  // // JSON
+  // $content = JSON.stringify({}, null, 2)
+
+  // 3. clash.meta
+
+  // 但是一般不需要这样用, 可参考 1. https://t.me/zhetengsha/1111 和 2. https://t.me/zhetengsha/1070
+
+  // let clashMetaProxies = await produceArtifact({
+  //     type: 'subscription',
+  //     name: 'sub',
+  //     platform: 'ClashMeta',
+  //     produceType: 'internal' // 'internal' produces an Array, otherwise produces a String( ProxyUtils.yaml.safeLoad('YAML String').proxies )
+  // }))
+
+  // // YAML
+  // $content = ProxyUtils.yaml.safeDump({})
+
+
+  // { $content, $files } will be passed to the next operator 
+  // $content is the final content of the file
   // flowUtils 为机场订阅流量信息处理工具
   // 可参考 https://t.me/zhetengsha/948
   // https://github.com/sub-store-org/Sub-Store/blob/31b6dd0507a9286d6ab834ec94ad3050f6bdc86b/backend/src/utils/download.js#L104