Merge pull request #3786 from Ultimaker/feature-backup-manager

Backup Manager and API

.gitignore

@@ -40,6 +40,7 @@ plugins/cura-siemensnx-plugin
 plugins/CuraBlenderPlugin
 plugins/CuraCloudPlugin
 plugins/CuraDrivePlugin
+plugins/CuraDrive
 plugins/CuraLiveScriptingPlugin
 plugins/CuraOpenSCADPlugin
 plugins/CuraPrintProfileCreator

cura/API/Backups.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2018 Ultimaker B.V.
+# Cura is released under the terms of the LGPLv3 or higher.
+from cura.Backups.BackupsManager import BackupsManager
+
+
+class Backups:
+    """
+    The backups API provides a version-proof bridge between Cura's BackupManager and plugins that hook into it.
+
+    Usage:
+        from cura.API import CuraAPI
+        api = CuraAPI()
+        api.backups.createBackup()
+        api.backups.restoreBackup(my_zip_file, {"cura_release": "3.1"})
+    """
+
+    manager = BackupsManager()  # Re-used instance of the backups manager.
+
+    def createBackup(self) -> (bytes, dict):
+        """
+        Create a new backup using the BackupsManager.
+        :return: Tuple containing a ZIP file with the backup data and a dict with meta data about the backup.
+        """
+        return self.manager.createBackup()
+
+    def restoreBackup(self, zip_file: bytes, meta_data: dict) -> None:
+        """
+        Restore a backup using the BackupManager.
+        :param zip_file: A ZIP file containing the actual backup data.
+        :param meta_data: Some meta data needed for restoring a backup, like the Cura version number.
+        """
+        return self.manager.restoreBackup(zip_file, meta_data)

cura/API/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2018 Ultimaker B.V.
+# Cura is released under the terms of the LGPLv3 or higher.
+from UM.PluginRegistry import PluginRegistry
+from cura.API.Backups import Backups
+
+
+class CuraAPI:
+    """
+    The official Cura API that plugins can use to interact with Cura.
+    Python does not technically prevent talking to other classes as well,
+    but this API provides a version-safe interface with proper deprecation warnings etc.
+    Usage of any other methods than the ones provided in this API can cause plugins to be unstable.
+    """
+
+    # For now we use the same API version to be consistent.
+    VERSION = PluginRegistry.APIVersion
+
+    # Backups API.
+    backups = Backups()

cura/AutoSave.py

@@ -3,17 +3,13 @@
 
 from PyQt5.QtCore import QTimer
 
-from UM.Extension import Extension
 from UM.Preferences import Preferences
-from UM.Application import Application
-from UM.Resources import Resources
 from UM.Logger import Logger
 
 
-class AutoSave(Extension):
-    def __init__(self):
-        super().__init__()
-
+class AutoSave:
+    def __init__(self, application):
+        self._application = application
         Preferences.getInstance().preferenceChanged.connect(self._triggerTimer)
 
         self._global_stack = None
@@ -26,29 +22,11 @@ class AutoSave(Extension):
 
         self._saving = False
 
-        # At this point, the Application instance has not finished its constructor call yet, so directly using something
-        # like Application.getInstance() is not correct. The initialisation now will only gets triggered after the
-        # application finishes its start up successfully.
-        self._init_timer = QTimer()
-        self._init_timer.setInterval(1000)
-        self._init_timer.setSingleShot(True)
-        self._init_timer.timeout.connect(self.initialize)
-        self._init_timer.start()
-
     def initialize(self):
         # only initialise if the application is created and has started
-        from cura.CuraApplication import CuraApplication
-        if not CuraApplication.Created:
-            self._init_timer.start()
-            return
-        if not CuraApplication.getInstance().started:
-            self._init_timer.start()
-            return
-
         self._change_timer.timeout.connect(self._onTimeout)
-        Application.getInstance().globalContainerStackChanged.connect(self._onGlobalStackChanged)
+        self._application.globalContainerStackChanged.connect(self._onGlobalStackChanged)
         self._onGlobalStackChanged()
-
         self._triggerTimer()
 
     def _triggerTimer(self, *args):
@@ -60,7 +38,7 @@ class AutoSave(Extension):
             self._global_stack.propertyChanged.disconnect(self._triggerTimer)
             self._global_stack.containersChanged.disconnect(self._triggerTimer)
 
-        self._global_stack = Application.getInstance().getGlobalContainerStack()
+        self._global_stack = self._application.getGlobalContainerStack()
 
         if self._global_stack:
             self._global_stack.propertyChanged.connect(self._triggerTimer)
@@ -70,8 +48,6 @@ class AutoSave(Extension):
         self._saving = True # To prevent the save process from triggering another autosave.
         Logger.log("d", "Autosaving preferences, instances and profiles")
 
-        Application.getInstance().saveSettings()
-
-        Preferences.getInstance().writeToFile(Resources.getStoragePath(Resources.Preferences, Application.getInstance().getApplicationName() + ".cfg"))
+        self._application.saveSettings()
 
         self._saving = False

cura/Backups/Backup.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2018 Ultimaker B.V.
+# Cura is released under the terms of the LGPLv3 or higher.
+import io
+import os
+import re
+
+import shutil
+
+from typing import Optional
+from zipfile import ZipFile, ZIP_DEFLATED, BadZipfile
+
+from UM import i18nCatalog
+from UM.Logger import Logger
+from UM.Message import Message
+from UM.Platform import Platform
+from UM.Resources import Resources
+from UM.Version import Version
+from cura.CuraApplication import CuraApplication
+
+
+class Backup:
+    """
+    The backup class holds all data about a backup.
+    It is also responsible for reading and writing the zip file to the user data folder.
+    """
+
+    # These files should be ignored when making a backup.
+    IGNORED_FILES = [r"cura\.log", r"plugins\.json", r"cache", r"__pycache__", r"\.qmlc", r"\.pyc"]
+
+    # Re-use translation catalog.
+    catalog = i18nCatalog("cura")
+
+    def __init__(self, zip_file: bytes = None, meta_data: dict = None):
+        self.zip_file = zip_file  # type: Optional[bytes]
+        self.meta_data = meta_data  # type: Optional[dict]
+
+    def makeFromCurrent(self) -> (bool, Optional[str]):
+        """
+        Create a backup from the current user config folder.
+        """
+        cura_release = CuraApplication.getInstance().getVersion()
+        version_data_dir = Resources.getDataStoragePath()
+
+        Logger.log("d", "Creating backup for Cura %s, using folder %s", cura_release, version_data_dir)
+
+        # Ensure all current settings are saved.
+        CuraApplication.getInstance().saveSettings()
+
+        # We copy the preferences file to the user data directory in Linux as it's in a different location there.
+        # When restoring a backup on Linux, we move it back.
+        if Platform.isLinux():
+            preferences_file_name = CuraApplication.getInstance().getApplicationName()
+            preferences_file = Resources.getPath(Resources.Preferences, "{}.cfg".format(preferences_file_name))
+            backup_preferences_file = os.path.join(version_data_dir, "{}.cfg".format(preferences_file_name))
+            Logger.log("d", "Copying preferences file from %s to %s", preferences_file, backup_preferences_file)
+            shutil.copyfile(preferences_file, backup_preferences_file)
+
+        # Create an empty buffer and write the archive to it.
+        buffer = io.BytesIO()
+        archive = self._makeArchive(buffer, version_data_dir)
+        files = archive.namelist()
+        
+        # Count the metadata items. We do this in a rather naive way at the moment.
+        machine_count = len([s for s in files if "machine_instances/" in s]) - 1
+        material_count = len([s for s in files if "materials/" in s]) - 1
+        profile_count = len([s for s in files if "quality_changes/" in s]) - 1
+        plugin_count = len([s for s in files if "plugin.json" in s])
+        
+        # Store the archive and metadata so the BackupManager can fetch them when needed.
+        self.zip_file = buffer.getvalue()
+        self.meta_data = {
+            "cura_release": cura_release,
+            "machine_count": str(machine_count),
+            "material_count": str(material_count),
+            "profile_count": str(profile_count),
+            "plugin_count": str(plugin_count)
+        }
+
+    def _makeArchive(self, buffer: "io.BytesIO", root_path: str) -> Optional[ZipFile]:
+        """
+        Make a full archive from the given root path with the given name.
+        :param root_path: The root directory to archive recursively.
+        :return: The archive as bytes.
+        """
+        ignore_string = re.compile("|".join(self.IGNORED_FILES))
+        try:
+            archive = ZipFile(buffer, "w", ZIP_DEFLATED)
+            for root, folders, files in os.walk(root_path):
+                for item_name in folders + files:
+                    absolute_path = os.path.join(root, item_name)
+                    if ignore_string.search(absolute_path):
+                        continue
+                    archive.write(absolute_path, absolute_path[len(root_path) + len(os.sep):])
+            archive.close()
+            return archive
+        except (IOError, OSError, BadZipfile) as error:
+            Logger.log("e", "Could not create archive from user data directory: %s", error)
+            self._showMessage(
+                self.catalog.i18nc("@info:backup_failed",
+                                   "Could not create archive from user data directory: {}".format(error)))
+            return None
+
+    def _showMessage(self, message: str) -> None:
+        """Show a UI message"""
+        Message(message, title=self.catalog.i18nc("@info:title", "Backup"), lifetime=30).show()
+
+    def restore(self) -> bool:
+        """
+        Restore this backups
+        :return: A boolean whether we had success or not.
+        """
+        if not self.zip_file or not self.meta_data or not self.meta_data.get("cura_release", None):
+            # We can restore without the minimum required information.
+            Logger.log("w", "Tried to restore a Cura backup without having proper data or meta data.")
+            self._showMessage(
+                self.catalog.i18nc("@info:backup_failed",
+                                   "Tried to restore a Cura backup without having proper data or meta data."))
+            return False
+
+        current_version = CuraApplication.getInstance().getVersion()
+        version_to_restore = self.meta_data.get("cura_release", "master")
+        if current_version != version_to_restore:
+            # Cannot restore version older or newer than current because settings might have changed.
+            # Restoring this will cause a lot of issues so we don't allow this for now.
+            self._showMessage(
+                self.catalog.i18nc("@info:backup_failed",
+                                   "Tried to restore a Cura backup that does not match your current version."))
+            return False
+
+        version_data_dir = Resources.getDataStoragePath()
+        archive = ZipFile(io.BytesIO(self.zip_file), "r")
+        extracted = self._extractArchive(archive, version_data_dir)
+
+        # Under Linux, preferences are stored elsewhere, so we copy the file to there.
+        if Platform.isLinux():
+            preferences_file_name = CuraApplication.getInstance().getApplicationName()
+            preferences_file = Resources.getPath(Resources.Preferences, "{}.cfg".format(preferences_file_name))
+            backup_preferences_file = os.path.join(version_data_dir, "{}.cfg".format(preferences_file_name))
+            Logger.log("d", "Moving preferences file from %s to %s", backup_preferences_file, preferences_file)
+            shutil.move(backup_preferences_file, preferences_file)
+
+        return extracted
+
+    @staticmethod
+    def _extractArchive(archive: "ZipFile", target_path: str) -> bool:
+        """
+        Extract the whole archive to the given target path.
+        :param archive: The archive as ZipFile.
+        :param target_path: The target path.
+        :return: A boolean whether we had success or not.
+        """
+        Logger.log("d", "Removing current data in location: %s", target_path)
+        Resources.factoryReset()
+        Logger.log("d", "Extracting backup to location: %s", target_path)
+        archive.extractall(target_path)
+        return True

cura/Backups/BackupsManager.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2018 Ultimaker B.V.
+# Cura is released under the terms of the LGPLv3 or higher.
+from typing import Optional
+
+from UM.Logger import Logger
+from cura.Backups.Backup import Backup
+from cura.CuraApplication import CuraApplication
+
+
+class BackupsManager:
+    """
+    The BackupsManager is responsible for managing the creating and restoring of backups.
+    Backups themselves are represented in a different class.
+    """
+
+    def createBackup(self) -> (Optional[bytes], Optional[dict]):
+        """
+        Get a backup of the current configuration.
+        :return: A Tuple containing a ZipFile (the actual backup) and a dict containing some meta data (like version).
+        """
+        self._disableAutoSave()
+        backup = Backup()
+        backup.makeFromCurrent()
+        self._enableAutoSave()
+        # We don't return a Backup here because we want plugins only to interact with our API and not full objects.
+        return backup.zip_file, backup.meta_data
+
+    def restoreBackup(self, zip_file: bytes, meta_data: dict) -> None:
+        """
+        Restore a backup from a given ZipFile.
+        :param zip_file: A bytes object containing the actual backup.
+        :param meta_data: A dict containing some meta data that is needed to restore the backup correctly.
+        """
+        if not meta_data.get("cura_release", None):
+            # If there is no "cura_release" specified in the meta data, we don't execute a backup restore.
+            Logger.log("w", "Tried to restore a backup without specifying a Cura version number.")
+            return
+
+        self._disableAutoSave()
+
+        backup = Backup(zip_file = zip_file, meta_data = meta_data)
+        restored = backup.restore()
+        if restored:
+            # At this point, Cura will need to restart for the changes to take effect.
+            # We don't want to store the data at this point as that would override the just-restored backup.
+            CuraApplication.getInstance().windowClosed(save_data=False)
+
+    def _disableAutoSave(self):
+        """Here we try to disable the auto-save plugin as it might interfere with restoring a backup."""
+        # TODO: Disable auto-save if possible.
+        CuraApplication.getInstance().setSaveDataEnabled(False)
+
+    def _enableAutoSave(self):
+        """Re-enable auto-save after we're done."""
+        # TODO: Enable auto-save if possible.
+        CuraApplication.getInstance().setSaveDataEnabled(True)

cura/CuraApplication.py

@@ -74,6 +74,7 @@ from cura.Settings.SimpleModeSettingsManager import SimpleModeSettingsManager
 
 from cura.Machines.VariantManager import VariantManager
 
+from .AutoSave import AutoSave
 from . import PlatformPhysics
 from . import BuildVolume
 from . import CameraAnimation
@@ -234,6 +235,8 @@ class CuraApplication(QtApplication):
         self._simple_mode_settings_manager = None
         self._cura_scene_controller = None
         self._machine_error_checker = None
+        self._auto_save = None
+        self._save_data_enabled = True
 
         self._additional_components = {} # Components to add to certain areas in the interface
 
@@ -497,15 +500,21 @@ class CuraApplication(QtApplication):
 
     showPrintMonitor = pyqtSignal(bool, arguments = ["show"])
 
+    def setSaveDataEnabled(self, enabled: bool) -> None:
+        self._save_data_enabled = enabled
+
     ##  Cura has multiple locations where instance containers need to be saved, so we need to handle this differently.
     #
     #   Note that the AutoSave plugin also calls this method.
     def saveSettings(self):
-        if not self.started: # Do not do saving during application start
+        if not self.started or not self._save_data_enabled:
+            # Do not do saving during application start or when data should not be safed on quit.
             return
-
         ContainerRegistry.getInstance().saveDirtyContainers()
 
+        Preferences.getInstance().writeToFile(Resources.getStoragePath(Resources.Preferences,
+                                                                       self._application_name + ".cfg"))
+
     def saveStack(self, stack):
         ContainerRegistry.getInstance().saveContainer(stack)
 
@@ -726,6 +735,9 @@ class CuraApplication(QtApplication):
         self._post_start_timer.timeout.connect(self._onPostStart)
         self._post_start_timer.start()
 
+        self._auto_save = AutoSave(self)
+        self._auto_save.initialize()
+
         self.exec_()
 
     def _onPostStart(self):
@@ -877,6 +889,9 @@ class CuraApplication(QtApplication):
 
         return super().event(event)
 
+    def getAutoSave(self):
+        return self._auto_save
+
     ##  Get print information (duration / material used)
     def getPrintInformation(self):
         return self._print_information

cura/Stages/CuraStage.py

@@ -1,9 +1,10 @@
 # Copyright (c) 2017 Ultimaker B.V.
 # Cura is released under the terms of the LGPLv3 or higher.
-from PyQt5.QtCore import pyqtProperty, QUrl, QObject
+from PyQt5.QtCore import pyqtProperty, QUrl
 
 from UM.Stage import Stage
 
+
 class CuraStage(Stage):
 
     def __init__(self, parent = None):

plugins/AutoSave/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2016 Ultimaker B.V.
-# Cura is released under the terms of the LGPLv3 or higher.
-
-from . import AutoSave
-
-from UM.i18n import i18nCatalog
-catalog = i18nCatalog("cura")
-
-def getMetaData():
-    return {}
-
-def register(app):
-    return { "extension": AutoSave.AutoSave() }

plugins/AutoSave/plugin.json

@@ -1,8 +0,0 @@
-{
-    "name": "Auto Save",
-    "author": "Ultimaker B.V.",
-    "version": "1.0.0",
-    "description": "Automatically saves Preferences, Machines and Profiles after changes.",
-    "api": 4,
-    "i18n-catalog": "cura"
-}

plugins/Toolbox/src/Toolbox.py

@@ -24,17 +24,26 @@ from .PackagesModel import PackagesModel
 
 i18n_catalog = i18nCatalog("cura")
 
+
 ##  The Toolbox class is responsible of communicating with the server through the API
 class Toolbox(QObject, Extension):
+
+    DEFAULT_PACKAGES_API_ROOT = "https://api.ultimaker.com"
+
     def __init__(self, parent=None) -> None:
         super().__init__(parent)
 
         self._application = Application.getInstance()
         self._package_manager = None
         self._plugin_registry = Application.getInstance().getPluginRegistry()
+        self._packages_api_root = self._getPackagesApiRoot()
         self._packages_version = self._getPackagesVersion()
         self._api_version = 1
-        self._api_url = "https://api.ultimaker.com/cura-packages/v{api_version}/cura/v{package_version}".format( api_version = self._api_version, package_version = self._packages_version)
+        self._api_url = "{api_root}/cura-packages/v{api_version}/cura/v{package_version}".format(
+            api_root = self._packages_api_root,
+            api_version = self._api_version,
+            package_version = self._packages_version
+        )
 
         # Network:
         self._get_packages_request = None
@@ -153,6 +162,15 @@ class Toolbox(QObject, Extension):
     def _onAppInitialized(self) -> None:
         self._package_manager = Application.getInstance().getCuraPackageManager()
 
+    # Get the API root for the packages API depending on Cura version settings.
+    def _getPackagesApiRoot(self) -> str:
+        if not hasattr(cura, "CuraVersion"):
+            return self.DEFAULT_PACKAGES_API_ROOT
+        if not hasattr(cura.CuraVersion, "CuraPackagesApiRoot"):
+            return self.DEFAULT_PACKAGES_API_ROOT
+        return cura.CuraVersion.CuraPackagesApiRoot
+
+    # Get the packages version depending on Cura version settings.
     def _getPackagesVersion(self) -> int:
         if not hasattr(cura, "CuraVersion"):
             return self._plugin_registry.APIVersion

plugins/USBPrinting/USBPrinterOutputDeviceManager.py

@@ -54,7 +54,7 @@ class USBPrinterOutputDeviceManager(QObject, OutputDevicePlugin):
         self._check_updates = True
         self._update_thread.start()
 
-    def stop(self):
+    def stop(self, store_data: bool = True):
         self._check_updates = False
 
     def _onConnectionStateChanged(self, serial_port):