From d92196da5346cba0e36d46882599f20c57c8e15a Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 12 Feb 2025 17:22:27 +0100 Subject: [PATCH 1/9] Exclude plugins available in Marketplace from backups. part of CURA-12156 --- cura/API/Backups.py | 15 +++---- cura/Backups/Backup.py | 35 ++++++++++++--- cura/Backups/BackupsManager.py | 9 ++-- plugins/CuraDrive/src/CreateBackupJob.py | 54 +++++++++++++++++++----- 4 files changed, 87 insertions(+), 26 deletions(-) diff --git a/cura/API/Backups.py b/cura/API/Backups.py index 1940d38a36..07001d22e2 100644 --- a/cura/API/Backups.py +++ b/cura/API/Backups.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Ultimaker B.V. +# Copyright (c) 2025 UltiMaker # Cura is released under the terms of the LGPLv3 or higher. from typing import Tuple, Optional, TYPE_CHECKING, Dict, Any @@ -9,14 +9,10 @@ if TYPE_CHECKING: class Backups: - """The back-ups API provides a version-proof bridge between Cura's - - BackupManager and plug-ins that hook into it. + """The back-ups API provides a version-proof bridge between Cura's BackupManager and plug-ins that hook into it. Usage: - .. code-block:: python - from cura.API import CuraAPI api = CuraAPI() api.backups.createBackup() @@ -26,13 +22,13 @@ class Backups: def __init__(self, application: "CuraApplication") -> None: self.manager = BackupsManager(application) - def createBackup(self) -> Tuple[Optional[bytes], Optional[Dict[str, Any]]]: + def createBackup(self, available_remote_plugins: frozenset[str] = frozenset()) -> Tuple[Optional[bytes], Optional[Dict[str, Any]]]: """Create a new back-up using the BackupsManager. :return: Tuple containing a ZIP file with the back-up data and a dict with metadata about the back-up. """ - return self.manager.createBackup() + return self.manager.createBackup(available_remote_plugins) def restoreBackup(self, zip_file: bytes, meta_data: Dict[str, Any]) -> None: """Restore a back-up using the BackupsManager. @@ -42,3 +38,6 @@ class Backups: """ return self.manager.restoreBackup(zip_file, meta_data) + + def shouldReinstallDownloadablePlugins(self) -> bool: + return self.manager.shouldReinstallDownloadablePlugins() diff --git a/cura/Backups/Backup.py b/cura/Backups/Backup.py index 19655df531..9f35e54ef1 100644 --- a/cura/Backups/Backup.py +++ b/cura/Backups/Backup.py @@ -1,5 +1,6 @@ -# Copyright (c) 2021 Ultimaker B.V. +# Copyright (c) 2025 UltiMaker # Cura is released under the terms of the LGPLv3 or higher. +import json import io import os @@ -13,6 +14,7 @@ from UM import i18nCatalog from UM.Logger import Logger from UM.Message import Message from UM.Platform import Platform +from UM.PluginRegistry import PluginRegistry from UM.Resources import Resources from UM.Version import Version @@ -30,10 +32,14 @@ class Backup: """These files should be ignored when making a backup.""" IGNORED_FOLDERS = [] # type: List[str] + """These folders should be ignored when making a backup.""" SECRETS_SETTINGS = ["general/ultimaker_auth_data"] """Secret preferences that need to obfuscated when making a backup of Cura""" + TO_INSTALL_FILE = "packages.json" + """File that contains the 'to_install' dictionary, that manages plugins to be installed on next startup.""" + catalog = i18nCatalog("cura") """Re-use translation catalog""" @@ -42,7 +48,7 @@ class Backup: self.zip_file = zip_file # type: Optional[bytes] self.meta_data = meta_data # type: Optional[Dict[str, str]] - def makeFromCurrent(self) -> None: + def makeFromCurrent(self, available_remote_plugins: frozenset[str] = frozenset()) -> None: """Create a back-up from the current user config folder.""" cura_release = self._application.getVersion() @@ -92,21 +98,40 @@ class Backup: # Restore the obfuscated settings self._illuminate(**secrets) - def _makeArchive(self, buffer: "io.BytesIO", root_path: str) -> Optional[ZipFile]: + def _fillToInstallsJson(self, file_path: str, reinstall_on_restore: dict[str, str], archive: ZipFile) -> None: + pass # TODO! + + def _findRedownloadablePlugins(self, available_remote_plugins: frozenset) -> dict[str, str]: + """ Find all plugins that should be able to be reinstalled from the Marketplace. + + :param plugins_path: Path to all plugins in the user-space. + :return: Set of all package-id's of plugins that can be reinstalled from the Marketplace. + """ + plugin_reg = PluginRegistry.getInstance() + id = "id" + return {v["location"]: v[id] for v in plugin_reg.getAllMetaData() + if v[id] in available_remote_plugins and not plugin_reg.isBundledPlugin(v[id])} + + def _makeArchive(self, buffer: "io.BytesIO", root_path: str, available_remote_plugins: frozenset) -> Optional[ZipFile]: """Make a full archive from the given root path with the given name. :param root_path: The root directory to archive recursively. :return: The archive as bytes. """ ignore_string = re.compile("|".join(self.IGNORED_FILES + self.IGNORED_FOLDERS)) + reinstall_instead_plugins = self._findRedownloadablePlugins(available_remote_plugins) try: archive = ZipFile(buffer, "w", ZIP_DEFLATED) - for root, folders, files in os.walk(root_path): + for root, folders, files in os.walk(root_path, topdown=True): + folders[:] = [f for f in folders if f not in reinstall_instead_plugins] for item_name in folders + files: absolute_path = os.path.join(root, item_name) if ignore_string.search(absolute_path): continue - archive.write(absolute_path, absolute_path[len(root_path) + len(os.sep):]) + if item_name == self.TO_INSTALL_FILE: + self._fillToInstallsJson(absolute_path, reinstall_instead_plugins, archive) + else: + archive.write(absolute_path, absolute_path[len(root_path) + len(os.sep):]) archive.close() return archive except (IOError, OSError, BadZipfile) as error: diff --git a/cura/Backups/BackupsManager.py b/cura/Backups/BackupsManager.py index 6c4670edb6..90dfc5e34e 100644 --- a/cura/Backups/BackupsManager.py +++ b/cura/Backups/BackupsManager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Ultimaker B.V. +# Copyright (c) 2025 UltiMaker # Cura is released under the terms of the LGPLv3 or higher. from typing import Dict, Optional, Tuple, TYPE_CHECKING @@ -22,7 +22,10 @@ class BackupsManager: def __init__(self, application: "CuraApplication") -> None: self._application = application - def createBackup(self) -> Tuple[Optional[bytes], Optional[Dict[str, str]]]: + def shouldReinstallDownloadablePlugins(self) -> bool: + return True + + def createBackup(self, available_remote_plugins: frozenset[str] = frozenset()) -> Tuple[Optional[bytes], Optional[Dict[str, str]]]: """ Get a back-up of the current configuration. @@ -31,7 +34,7 @@ class BackupsManager: self._disableAutoSave() backup = Backup(self._application) - backup.makeFromCurrent() + backup.makeFromCurrent(available_remote_plugins if self.shouldReinstallDownloadablePlugins() else frozenset()) self._enableAutoSave() # We don't return a Backup here because we want plugins only to interact with our API and not full objects. return backup.zip_file, backup.meta_data diff --git a/plugins/CuraDrive/src/CreateBackupJob.py b/plugins/CuraDrive/src/CreateBackupJob.py index 7d772769ed..cdd1d569c7 100644 --- a/plugins/CuraDrive/src/CreateBackupJob.py +++ b/plugins/CuraDrive/src/CreateBackupJob.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Ultimaker B.V. +# Copyright (c) 2025 UltiMaker # Cura is released under the terms of the LGPLv3 or higher. import json import threading @@ -13,11 +13,14 @@ from UM.Message import Message from UM.TaskManagement.HttpRequestManager import HttpRequestManager from UM.TaskManagement.HttpRequestScope import JsonDecoratorScope from UM.i18n import i18nCatalog +from cura.ApplicationMetadata import CuraSDKVersion from cura.CuraApplication import CuraApplication from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope +import cura.UltimakerCloud.UltimakerCloudConstants as UltimakerCloudConstants catalog = i18nCatalog("cura") +PACKAGES_URL = f"{UltimakerCloudConstants.CuraCloudAPIRoot}/cura-packages/v{UltimakerCloudConstants.CuraCloudAPIVersion}/cura/v{CuraSDKVersion}/packages" class CreateBackupJob(Job): """Creates backup zip, requests upload url and uploads the backup file to cloud storage.""" @@ -40,23 +43,54 @@ class CreateBackupJob(Job): self._job_done = threading.Event() """Set when the job completes. Does not indicate success.""" self.backup_upload_error_message = "" - """After the job completes, an empty string indicates success. Othrerwise, the value is a translated message.""" + """After the job completes, an empty string indicates success. Otherwise, the value is a translated message.""" + + def _setPluginFetchErrorMessage(self, error_msg: str) -> None: + Logger.error(f"Fetching plugins for backup resulted in error: {error_msg}") + self.backup_upload_error_message = "Couldn't update currently available plugins, backup stopped." + self._upload_message.hide() + self._job_done.set() def run(self) -> None: - upload_message = Message(catalog.i18nc("@info:backup_status", "Creating your backup..."), + self._upload_message = Message(catalog.i18nc("@info:backup_status", "Fetch re-downloadable package-ids..."), title = self.MESSAGE_TITLE, progress = -1) - upload_message.show() + self._upload_message.show() + CuraApplication.getInstance().processEvents() + + if CuraApplication.getInstance().getCuraAPI().backups.shouldReinstallDownloadablePlugins(): + request_url = f"{PACKAGES_URL}?package_type=plugin" + scope = JsonDecoratorScope(UltimakerCloudScope(CuraApplication.getInstance())) + HttpRequestManager.getInstance().get( + request_url, + scope=scope, + callback=self._continueRun, + error_callback=lambda reply, error: self._setPluginFetchErrorMessage(str(error)), + ) + else: + self._continueRun() + + def _continueRun(self, reply: "QNetworkReply" = None) -> None: + if reply is not None: + response_data = HttpRequestManager.readJSON(reply) + if "data" not in response_data: + self._setPluginFetchErrorMessage(f"Missing 'data' from response. Keys in response: {response_data.keys()}") + return + available_remote_plugins = frozenset({v["package_id"] for v in response_data["data"]}) + else: + available_remote_plugins = frozenset() + + self._upload_message.setText(catalog.i18nc("@info:backup_status", "Creating your backup...")) CuraApplication.getInstance().processEvents() cura_api = CuraApplication.getInstance().getCuraAPI() - self._backup_zip, backup_meta_data = cura_api.backups.createBackup() + self._backup_zip, backup_meta_data = cura_api.backups.createBackup(available_remote_plugins) if not self._backup_zip or not backup_meta_data: self.backup_upload_error_message = catalog.i18nc("@info:backup_status", "There was an error while creating your backup.") - upload_message.hide() + self._upload_message.hide() return - upload_message.setText(catalog.i18nc("@info:backup_status", "Uploading your backup...")) + self._upload_message.setText(catalog.i18nc("@info:backup_status", "Uploading your backup...")) CuraApplication.getInstance().processEvents() # Create an upload entry for the backup. @@ -66,11 +100,11 @@ class CreateBackupJob(Job): self._job_done.wait() if self.backup_upload_error_message == "": - upload_message.setText(catalog.i18nc("@info:backup_status", "Your backup has finished uploading.")) - upload_message.setProgress(None) # Hide progress bar + self._upload_message.setText(catalog.i18nc("@info:backup_status", "Your backup has finished uploading.")) + self._upload_message.setProgress(None) # Hide progress bar else: # some error occurred. This error is presented to the user by DrivePluginExtension - upload_message.hide() + self._upload_message.hide() def _requestUploadSlot(self, backup_metadata: Dict[str, Any], backup_size: int) -> None: """Request a backup upload slot from the API. From d167e3f28ed8843786414faa5a0331a5d3ee33fa Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Thu, 27 Feb 2025 16:52:27 +0100 Subject: [PATCH 2/9] Work in progress on pulling plugins out of the backups. It's now in a state where it can actually upload ... something (that should work). Not tested the restore yet. I did run into trouble with the max concurrent requests, which I had to up to [more than 4, now on 8] to get it to work -- I'm not sure if I'm just working around a bug here, or if that's expected behaviour. part of CURA-12156 --- cura/API/Backups.py | 4 +- cura/Backups/Backup.py | 67 ++++++++++++------ cura/Backups/BackupsManager.py | 5 +- plugins/CuraDrive/src/CreateBackupJob.py | 4 ++ plugins/CuraDrive/src/RestoreBackupJob.py | 82 +++++++++++++++++++++-- 5 files changed, 132 insertions(+), 30 deletions(-) diff --git a/cura/API/Backups.py b/cura/API/Backups.py index 07001d22e2..a52dcbfb6b 100644 --- a/cura/API/Backups.py +++ b/cura/API/Backups.py @@ -30,14 +30,14 @@ class Backups: return self.manager.createBackup(available_remote_plugins) - def restoreBackup(self, zip_file: bytes, meta_data: Dict[str, Any]) -> None: + def restoreBackup(self, zip_file: bytes, meta_data: Dict[str, Any], auto_close: bool = True) -> None: """Restore a back-up using the BackupsManager. :param zip_file: A ZIP file containing the actual back-up data. :param meta_data: Some metadata needed for restoring a back-up, like the Cura version number. """ - return self.manager.restoreBackup(zip_file, meta_data) + return self.manager.restoreBackup(zip_file, meta_data, auto_close=auto_close) def shouldReinstallDownloadablePlugins(self) -> bool: return self.manager.shouldReinstallDownloadablePlugins() diff --git a/cura/Backups/Backup.py b/cura/Backups/Backup.py index 9f35e54ef1..1163169b94 100644 --- a/cura/Backups/Backup.py +++ b/cura/Backups/Backup.py @@ -1,5 +1,7 @@ # Copyright (c) 2025 UltiMaker # Cura is released under the terms of the LGPLv3 or higher. +import tempfile + import json import io @@ -8,7 +10,7 @@ import re import shutil from copy import deepcopy from zipfile import ZipFile, ZIP_DEFLATED, BadZipfile -from typing import Dict, Optional, TYPE_CHECKING, List +from typing import Callable, Dict, Optional, TYPE_CHECKING, List from UM import i18nCatalog from UM.Logger import Logger @@ -37,9 +39,6 @@ class Backup: SECRETS_SETTINGS = ["general/ultimaker_auth_data"] """Secret preferences that need to obfuscated when making a backup of Cura""" - TO_INSTALL_FILE = "packages.json" - """File that contains the 'to_install' dictionary, that manages plugins to be installed on next startup.""" - catalog = i18nCatalog("cura") """Re-use translation catalog""" @@ -74,7 +73,7 @@ class Backup: # Create an empty buffer and write the archive to it. buffer = io.BytesIO() - archive = self._makeArchive(buffer, version_data_dir) + archive = self._makeArchive(buffer, version_data_dir, available_remote_plugins) if archive is None: return files = archive.namelist() @@ -83,9 +82,7 @@ class Backup: machine_count = max(len([s for s in files if "machine_instances/" in s]) - 1, 0) # If people delete their profiles but not their preferences, it can still make a backup, and report -1 profiles. Server crashes on this. material_count = max(len([s for s in files if "materials/" in s]) - 1, 0) profile_count = max(len([s for s in files if "quality_changes/" in s]) - 1, 0) - # We don't store plugins anymore, since if you can make backups, you have an account (and the plugins are - # on the marketplace anyway) - plugin_count = 0 + plugin_count = len([s for s in files if "plugin.json" in s]) # Store the archive and metadata so the BackupManager can fetch them when needed. self.zip_file = buffer.getvalue() self.meta_data = { @@ -98,19 +95,43 @@ class Backup: # Restore the obfuscated settings self._illuminate(**secrets) - def _fillToInstallsJson(self, file_path: str, reinstall_on_restore: dict[str, str], archive: ZipFile) -> None: - pass # TODO! + def _fillToInstallsJson(self, file_path: str, reinstall_on_restore: frozenset[str], add_to_archive: Callable[[str], None]) -> Optional[str]: + """ Moves all plugin-data (in a config-file) for plugins that could be (re)installed from the Marketplace from + 'installed' to 'to_installs' before adding that file to the archive. - def _findRedownloadablePlugins(self, available_remote_plugins: frozenset) -> dict[str, str]: + Note that the 'filename'-entry in the package-data (of the plugins) might not be valid anymore on restore. + We'll replace it on restore instead, as that's the time when the new package is downloaded. + + :param file_path: Absolute path to the packages-file. + :param reinstall_on_restore: A set of plugins that _can_ be reinstalled from the Marketplace. + :param add_to_archive: A function/lambda that takes a filename and adds it to the archive. + """ + with open(file_path, "r") as file: + data = json.load(file) + reinstall, keep_in = {}, {} + for install_id, install_info in data["installed"].items(): + (reinstall if install_id in reinstall_on_restore else keep_in)[install_id] = install_info + data["installed"] = keep_in + data["to_install"].update(reinstall) + if data is not None: + tmpfile = tempfile.NamedTemporaryFile(delete=False) + with open(tmpfile.name, "w") as outfile: + json.dump(data, outfile) + add_to_archive(tmpfile.name) + return tmpfile.name + return None + + def _findRedownloadablePlugins(self, available_remote_plugins: frozenset) -> (frozenset[str], frozenset[str]): """ Find all plugins that should be able to be reinstalled from the Marketplace. :param plugins_path: Path to all plugins in the user-space. - :return: Set of all package-id's of plugins that can be reinstalled from the Marketplace. + :return: Tuple of a set of plugin-ids and a set of plugin-paths. """ plugin_reg = PluginRegistry.getInstance() id = "id" - return {v["location"]: v[id] for v in plugin_reg.getAllMetaData() - if v[id] in available_remote_plugins and not plugin_reg.isBundledPlugin(v[id])} + plugins = [v for v in plugin_reg.getAllMetaData() + if v[id] in available_remote_plugins and not plugin_reg.isBundledPlugin(v[id])] + return frozenset([v[id] for v in plugins]), frozenset([v["location"] for v in plugins]) def _makeArchive(self, buffer: "io.BytesIO", root_path: str, available_remote_plugins: frozenset) -> Optional[ZipFile]: """Make a full archive from the given root path with the given name. @@ -119,20 +140,28 @@ class Backup: :return: The archive as bytes. """ ignore_string = re.compile("|".join(self.IGNORED_FILES + self.IGNORED_FOLDERS)) - reinstall_instead_plugins = self._findRedownloadablePlugins(available_remote_plugins) + reinstall_instead_ids, reinstall_instead_paths = self._findRedownloadablePlugins(available_remote_plugins) + tmpfiles = [] try: archive = ZipFile(buffer, "w", ZIP_DEFLATED) + add_path_to_archive = lambda path: archive.write(path, path[len(root_path) + len(os.sep):]) for root, folders, files in os.walk(root_path, topdown=True): - folders[:] = [f for f in folders if f not in reinstall_instead_plugins] + folders[:] = [f for f in folders if f not in reinstall_instead_paths] for item_name in folders + files: absolute_path = os.path.join(root, item_name) if ignore_string.search(absolute_path): continue - if item_name == self.TO_INSTALL_FILE: - self._fillToInstallsJson(absolute_path, reinstall_instead_plugins, archive) + if item_name == "packages.json": + tmpfiles.append( + self._fillToInstallsJson(absolute_path, reinstall_instead_ids, add_path_to_archive)) else: - archive.write(absolute_path, absolute_path[len(root_path) + len(os.sep):]) + add_path_to_archive(absolute_path) archive.close() + for tmpfile_path in tmpfiles: + try: + os.remove(tmpfile_path) + except IOError as ex: + Logger.warning(f"Couldn't remove temporary file '{tmpfile_path}' because '{ex}'.") return archive except (IOError, OSError, BadZipfile) as error: Logger.log("e", "Could not create archive from user data directory: %s", error) diff --git a/cura/Backups/BackupsManager.py b/cura/Backups/BackupsManager.py index 90dfc5e34e..67d6c84601 100644 --- a/cura/Backups/BackupsManager.py +++ b/cura/Backups/BackupsManager.py @@ -39,12 +39,13 @@ class BackupsManager: # We don't return a Backup here because we want plugins only to interact with our API and not full objects. return backup.zip_file, backup.meta_data - def restoreBackup(self, zip_file: bytes, meta_data: Dict[str, str]) -> None: + def restoreBackup(self, zip_file: bytes, meta_data: Dict[str, str], auto_close: bool = True) -> None: """ Restore a back-up from a given ZipFile. :param zip_file: A bytes object containing the actual back-up. :param meta_data: A dict containing some metadata that is needed to restore the back-up correctly. + :param auto_close: Normally, Cura will need to close immediately after restoring the back-up. """ if not meta_data.get("cura_release", None): @@ -57,7 +58,7 @@ class BackupsManager: backup = Backup(self._application, zip_file = zip_file, meta_data = meta_data) restored = backup.restore() - if restored: + if restored and auto_close: # At this point, Cura will need to restart for the changes to take effect. # We don't want to store the data at this point as that would override the just-restored backup. self._application.windowClosed(save_data = False) diff --git a/plugins/CuraDrive/src/CreateBackupJob.py b/plugins/CuraDrive/src/CreateBackupJob.py index cdd1d569c7..6297af305f 100644 --- a/plugins/CuraDrive/src/CreateBackupJob.py +++ b/plugins/CuraDrive/src/CreateBackupJob.py @@ -118,6 +118,8 @@ class CreateBackupJob(Job): } }).encode() + CuraApplication.getInstance().processEvents() # Needed?? + HttpRequestManager.getInstance().put( self._api_backup_url, data = payload, @@ -125,6 +127,8 @@ class CreateBackupJob(Job): error_callback = self._onUploadSlotCompleted, scope = self._json_cloud_scope) + CuraApplication.getInstance().processEvents() # Needed?? + def _onUploadSlotCompleted(self, reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None: if HttpRequestManager.safeHttpStatus(reply) >= 300: replyText = HttpRequestManager.readText(reply) diff --git a/plugins/CuraDrive/src/RestoreBackupJob.py b/plugins/CuraDrive/src/RestoreBackupJob.py index 54c94b389e..c5fd1fceae 100644 --- a/plugins/CuraDrive/src/RestoreBackupJob.py +++ b/plugins/CuraDrive/src/RestoreBackupJob.py @@ -1,8 +1,12 @@ # Copyright (c) 2021 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. +import tempfile + +import json import base64 import hashlib +import os import threading from tempfile import NamedTemporaryFile from typing import Optional, Any, Dict @@ -12,9 +16,16 @@ from PyQt6.QtNetwork import QNetworkReply, QNetworkRequest from UM.Job import Job from UM.Logger import Logger from UM.PackageManager import catalog +from UM.Resources import Resources from UM.TaskManagement.HttpRequestManager import HttpRequestManager -from cura.CuraApplication import CuraApplication +from UM.Version import Version +from cura.ApplicationMetadata import CuraSDKVersion +from cura.CuraApplication import CuraApplication +from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope +import cura.UltimakerCloud.UltimakerCloudConstants as UltimakerCloudConstants + +PACKAGES_URL = f"{UltimakerCloudConstants.CuraCloudAPIRoot}/cura-packages/v{UltimakerCloudConstants.CuraCloudAPIVersion}/cura/v{CuraSDKVersion}/packages" class RestoreBackupJob(Job): """Downloads a backup and overwrites local configuration with the backup. @@ -60,8 +71,8 @@ class RestoreBackupJob(Job): # We store the file in a temporary path fist to ensure integrity. try: - temporary_backup_file = NamedTemporaryFile(delete = False) - with open(temporary_backup_file.name, "wb") as write_backup: + self._temporary_backup_file = NamedTemporaryFile(delete = False) + with open(self._temporary_backup_file.name, "wb") as write_backup: app = CuraApplication.getInstance() bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) while bytes_read: @@ -74,18 +85,75 @@ class RestoreBackupJob(Job): self._job_done.set() return - if not self._verifyMd5Hash(temporary_backup_file.name, self._backup.get("md5_hash", "")): + if not self._verifyMd5Hash(self._temporary_backup_file.name, self._backup.get("md5_hash", "")): # Don't restore the backup if the MD5 hashes do not match. # This can happen if the download was interrupted. Logger.log("w", "Remote and local MD5 hashes do not match, not restoring backup.") self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE # Tell Cura to place the backup back in the user data folder. - with open(temporary_backup_file.name, "rb") as read_backup: + metadata = self._backup.get("metadata", {}) + with open(self._temporary_backup_file.name, "rb") as read_backup: cura_api = CuraApplication.getInstance().getCuraAPI() - cura_api.backups.restoreBackup(read_backup.read(), self._backup.get("metadata", {})) + cura_api.backups.restoreBackup(read_backup.read(), metadata, auto_close=False) - self._job_done.set() + # Read packages data-file, to get the 'to_install' plugin-ids. + version_to_restore = Version(metadata.get("cura_release", "dev")) + version_str = f"{version_to_restore.getMajor()}.{version_to_restore.getMinor()}" + packages_path = os.path.abspath(os.path.join(os.path.abspath( + Resources.getConfigStoragePath()), "..", version_str, "packages.json")) + if not os.path.exists(packages_path): + self._job_done.set() + return + + to_install = set() + try: + with open(packages_path, "r") as packages_file: + packages_json = json.load(packages_file) + if "to_install" in packages_json and "package_id" in packages_json["to_install"]: + to_install.add(packages_json["to_install"]["package_id"]) + except IOError as ex: + pass # TODO! (log + message) + + if len(to_install) < 1: + self._job_done.set() + return + + # Download all re-installable plugins packages, so they can be put back on start-up. + redownload_errors = [] + def packageDownloadCallback(package_id: str, msg: "QNetworkReply", err: "QNetworkReply.NetworkError" = None) -> None: + if err is not None or HttpRequestManager.safeHttpStatus(msg) != 200: + redownload_errors.append(err) + to_install.remove(package_id) + + try: + with tempfile.NamedTemporaryFile(mode="wb+", suffix=".curapackage") as temp_file: + bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) + while bytes_read: + temp_file.write(bytes_read) + bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) + # self._app.processEvents() + # self._progress[package_id]["file_written"] = temp_file.name + if not CuraApplication.getInstance().getPackageManager().installPackage(temp_file.name): + redownload_errors.append(f"Couldn't install package '{package_id}'.") + except IOError as ex: + redownload_errors.append(f"Couldn't read package '{package_id}' because '{ex}'.") + + if len(to_install) < 1: + if len(redownload_errors) == 0: + self._job_done.set() + else: + print("|".join(redownload_errors)) # TODO: Message / Log instead. + self._job_done.set() # NOTE: Set job probably not the right call here... (depends on wether or not that in the end closes the app or not...) + + self._package_download_scope = UltimakerCloudScope(CuraApplication.getInstance()) + for package_id in to_install: + HttpRequestManager.getInstance().get( + f"{PACKAGES_URL}/{package_id}/download", + scope=self._package_download_scope, + callback=lambda msg: packageDownloadCallback(package_id, msg), + error_callback=lambda msg, err: packageDownloadCallback(package_id, msg, err) + ) @staticmethod def _verifyMd5Hash(file_path: str, known_hash: str) -> bool: From b295ca7d041776d2f7c06be4146fbac0e043424f Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 1 Apr 2025 15:10:27 +0200 Subject: [PATCH 3/9] Workaround for process-events during multiple http requests. Would not work otherwise, but that should have been true in the old situation then as well? CURA-12156 --- plugins/CuraDrive/src/CreateBackupJob.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/CuraDrive/src/CreateBackupJob.py b/plugins/CuraDrive/src/CreateBackupJob.py index 6297af305f..67083d2e83 100644 --- a/plugins/CuraDrive/src/CreateBackupJob.py +++ b/plugins/CuraDrive/src/CreateBackupJob.py @@ -98,7 +98,12 @@ class CreateBackupJob(Job): backup_meta_data["description"] = "{}.backup.{}.cura.zip".format(timestamp, backup_meta_data["cura_release"]) self._requestUploadSlot(backup_meta_data, len(self._backup_zip)) - self._job_done.wait() + # Note: One 'process events' call wasn't enough with the changed situation somehow. + active_done_check = False + while not active_done_check: + CuraApplication.getInstance().processEvents() + active_done_check = self._job_done.wait(0.02) + if self.backup_upload_error_message == "": self._upload_message.setText(catalog.i18nc("@info:backup_status", "Your backup has finished uploading.")) self._upload_message.setProgress(None) # Hide progress bar @@ -117,9 +122,6 @@ class CreateBackupJob(Job): "metadata": backup_metadata } }).encode() - - CuraApplication.getInstance().processEvents() # Needed?? - HttpRequestManager.getInstance().put( self._api_backup_url, data = payload, @@ -127,8 +129,6 @@ class CreateBackupJob(Job): error_callback = self._onUploadSlotCompleted, scope = self._json_cloud_scope) - CuraApplication.getInstance().processEvents() # Needed?? - def _onUploadSlotCompleted(self, reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None: if HttpRequestManager.safeHttpStatus(reply) >= 300: replyText = HttpRequestManager.readText(reply) From c857dab0f76196a803871da1b10a331458b262ee Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 1 Apr 2025 16:47:17 +0200 Subject: [PATCH 4/9] Logging (mostly on errors). CURA-12156 --- plugins/CuraDrive/src/RestoreBackupJob.py | 42 ++++++++++++++--------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/plugins/CuraDrive/src/RestoreBackupJob.py b/plugins/CuraDrive/src/RestoreBackupJob.py index c5fd1fceae..e7cd66daf8 100644 --- a/plugins/CuraDrive/src/RestoreBackupJob.py +++ b/plugins/CuraDrive/src/RestoreBackupJob.py @@ -1,11 +1,8 @@ -# Copyright (c) 2021 Ultimaker B.V. +# Copyright (c) 2025 UltiMaker # Cura is released under the terms of the LGPLv3 or higher. -import tempfile - -import json - import base64 import hashlib +import json import os import threading from tempfile import NamedTemporaryFile @@ -49,7 +46,6 @@ class RestoreBackupJob(Job): self.restore_backup_error_message = "" def run(self) -> None: - url = self._backup.get("download_url") assert url is not None @@ -59,7 +55,11 @@ class RestoreBackupJob(Job): error_callback = self._onRestoreRequestCompleted ) - self._job_done.wait() # A job is considered finished when the run function completes + # Note: Just to be sure, use the same structure here as in CreateBackupJob. + active_done_check = False + while not active_done_check: + CuraApplication.getInstance().processEvents() + active_done_check = self._job_done.wait(0.02) def _onRestoreRequestCompleted(self, reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None: if not HttpRequestManager.replyIndicatesSuccess(reply, error): @@ -80,7 +80,7 @@ class RestoreBackupJob(Job): bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) app.processEvents() except EnvironmentError as e: - Logger.log("e", f"Unable to save backed up files due to computer limitations: {str(e)}") + Logger.error(f"Unable to save backed up files due to computer limitations: {str(e)}") self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE self._job_done.set() return @@ -88,8 +88,10 @@ class RestoreBackupJob(Job): if not self._verifyMd5Hash(self._temporary_backup_file.name, self._backup.get("md5_hash", "")): # Don't restore the backup if the MD5 hashes do not match. # This can happen if the download was interrupted. - Logger.log("w", "Remote and local MD5 hashes do not match, not restoring backup.") + Logger.error("Remote and local MD5 hashes do not match, not restoring backup.") self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE + self._job_done.set() + return # Tell Cura to place the backup back in the user data folder. metadata = self._backup.get("metadata", {}) @@ -103,6 +105,8 @@ class RestoreBackupJob(Job): packages_path = os.path.abspath(os.path.join(os.path.abspath( Resources.getConfigStoragePath()), "..", version_str, "packages.json")) if not os.path.exists(packages_path): + Logger.error(f"Can't find path '{packages_path}' to tell what packages should be redownloaded.") + self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE self._job_done.set() return @@ -113,9 +117,13 @@ class RestoreBackupJob(Job): if "to_install" in packages_json and "package_id" in packages_json["to_install"]: to_install.add(packages_json["to_install"]["package_id"]) except IOError as ex: - pass # TODO! (log + message) + Logger.error(f"Couldn't open '{packages_path}' because '{str(ex)}' to get packages to re-install.") + self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE + self._job_done.set() + return if len(to_install) < 1: + Logger.info("No packages to reinstall, early out.") self._job_done.set() return @@ -127,24 +135,26 @@ class RestoreBackupJob(Job): to_install.remove(package_id) try: - with tempfile.NamedTemporaryFile(mode="wb+", suffix=".curapackage") as temp_file: + with NamedTemporaryFile(mode="wb+", suffix=".curapackage") as temp_file: bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) while bytes_read: temp_file.write(bytes_read) bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) - # self._app.processEvents() - # self._progress[package_id]["file_written"] = temp_file.name + CuraApplication.getInstance().processEvents() if not CuraApplication.getInstance().getPackageManager().installPackage(temp_file.name): redownload_errors.append(f"Couldn't install package '{package_id}'.") except IOError as ex: - redownload_errors.append(f"Couldn't read package '{package_id}' because '{ex}'.") + redownload_errors.append(f"Couldn't process package '{package_id}' because '{ex}'.") if len(to_install) < 1: if len(redownload_errors) == 0: + Logger.info("All packages redownloaded!") self._job_done.set() else: - print("|".join(redownload_errors)) # TODO: Message / Log instead. - self._job_done.set() # NOTE: Set job probably not the right call here... (depends on wether or not that in the end closes the app or not...) + msgs = "\n - ".join(redownload_errors) + Logger.error(f"Couldn't re-install at least one package(s) because: {msgs}") + self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE + self._job_done.set() self._package_download_scope = UltimakerCloudScope(CuraApplication.getInstance()) for package_id in to_install: From 6458c17de5b2a92a3179e2747b75c6ea394f138e Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 2 Apr 2025 13:09:21 +0200 Subject: [PATCH 5/9] Save 'pluginless' bakcup correctly. - Fix: Save the tempfile to the archive under the 'original' name (it is a rewrite of) instead of saving it to the archive under it's own name, which skipped the original file completely in a sense (the info was there, but as a tempfile). - Fix: Also make sure the correct folders where ignored, as reinstall paths where the complete path, not the basename. part of CURA-12156 --- cura/Backups/Backup.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cura/Backups/Backup.py b/cura/Backups/Backup.py index 1163169b94..a409c4c689 100644 --- a/cura/Backups/Backup.py +++ b/cura/Backups/Backup.py @@ -95,7 +95,7 @@ class Backup: # Restore the obfuscated settings self._illuminate(**secrets) - def _fillToInstallsJson(self, file_path: str, reinstall_on_restore: frozenset[str], add_to_archive: Callable[[str], None]) -> Optional[str]: + def _fillToInstallsJson(self, file_path: str, reinstall_on_restore: frozenset[str], add_to_archive: Callable[[str, str], None]) -> Optional[str]: """ Moves all plugin-data (in a config-file) for plugins that could be (re)installed from the Marketplace from 'installed' to 'to_installs' before adding that file to the archive. @@ -104,7 +104,7 @@ class Backup: :param file_path: Absolute path to the packages-file. :param reinstall_on_restore: A set of plugins that _can_ be reinstalled from the Marketplace. - :param add_to_archive: A function/lambda that takes a filename and adds it to the archive. + :param add_to_archive: A function/lambda that takes a filename and adds it to the archive (as the 2nd name). """ with open(file_path, "r") as file: data = json.load(file) @@ -117,7 +117,7 @@ class Backup: tmpfile = tempfile.NamedTemporaryFile(delete=False) with open(tmpfile.name, "w") as outfile: json.dump(data, outfile) - add_to_archive(tmpfile.name) + add_to_archive(tmpfile.name, file_path) return tmpfile.name return None @@ -144,18 +144,17 @@ class Backup: tmpfiles = [] try: archive = ZipFile(buffer, "w", ZIP_DEFLATED) - add_path_to_archive = lambda path: archive.write(path, path[len(root_path) + len(os.sep):]) + add_path_to_archive = lambda path, alt_path: archive.write(path, alt_path[len(root_path) + len(os.sep):]) for root, folders, files in os.walk(root_path, topdown=True): - folders[:] = [f for f in folders if f not in reinstall_instead_paths] for item_name in folders + files: absolute_path = os.path.join(root, item_name) - if ignore_string.search(absolute_path): + if ignore_string.search(absolute_path) or any([absolute_path.startswith(x) for x in reinstall_instead_paths]): continue if item_name == "packages.json": tmpfiles.append( self._fillToInstallsJson(absolute_path, reinstall_instead_ids, add_path_to_archive)) else: - add_path_to_archive(absolute_path) + add_path_to_archive(absolute_path, absolute_path) archive.close() for tmpfile_path in tmpfiles: try: From 1f4f432d49c2649521fde3233de4943bcacfb080 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 2 Apr 2025 13:13:25 +0200 Subject: [PATCH 6/9] Restore Backups: Fix reading which packages need to be reinstalled. Also save the version, so we can get the correct url when redownloading the package (see next commits). part of CURA-12156 --- plugins/CuraDrive/src/RestoreBackupJob.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/CuraDrive/src/RestoreBackupJob.py b/plugins/CuraDrive/src/RestoreBackupJob.py index e7cd66daf8..4127df7aa6 100644 --- a/plugins/CuraDrive/src/RestoreBackupJob.py +++ b/plugins/CuraDrive/src/RestoreBackupJob.py @@ -110,12 +110,17 @@ class RestoreBackupJob(Job): self._job_done.set() return - to_install = set() + to_install = {} try: with open(packages_path, "r") as packages_file: packages_json = json.load(packages_file) - if "to_install" in packages_json and "package_id" in packages_json["to_install"]: - to_install.add(packages_json["to_install"]["package_id"]) + if "to_install" in packages_json: + for package_data in packages_json["to_install"].values(): + if "package_info" not in package_data: + continue + package_info = package_data["package_info"] + if "package_id" in package_info and "sdk_version_semver" in package_info: + to_install[package_info["package_id"]] = package_info["sdk_version_semver"] except IOError as ex: Logger.error(f"Couldn't open '{packages_path}' because '{str(ex)}' to get packages to re-install.") self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE @@ -132,7 +137,7 @@ class RestoreBackupJob(Job): def packageDownloadCallback(package_id: str, msg: "QNetworkReply", err: "QNetworkReply.NetworkError" = None) -> None: if err is not None or HttpRequestManager.safeHttpStatus(msg) != 200: redownload_errors.append(err) - to_install.remove(package_id) + del to_install[package_id] try: with NamedTemporaryFile(mode="wb+", suffix=".curapackage") as temp_file: From 43d9e1d522e449c9880428f7d7fd0f8f3a111073 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 2 Apr 2025 13:15:35 +0200 Subject: [PATCH 7/9] Restore Backups: Fix handling the (re)downloaded tempfile. Was opening the tempfile for handling when it was still open for writing. Also the wrong net-reply got used (reply instead of msg). part of CURA-12156 --- plugins/CuraDrive/src/RestoreBackupJob.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/plugins/CuraDrive/src/RestoreBackupJob.py b/plugins/CuraDrive/src/RestoreBackupJob.py index 4127df7aa6..aa48ca5c18 100644 --- a/plugins/CuraDrive/src/RestoreBackupJob.py +++ b/plugins/CuraDrive/src/RestoreBackupJob.py @@ -140,14 +140,15 @@ class RestoreBackupJob(Job): del to_install[package_id] try: - with NamedTemporaryFile(mode="wb+", suffix=".curapackage") as temp_file: - bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) + with NamedTemporaryFile(mode="wb", suffix=".curapackage", delete=False) as temp_file: + bytes_read = msg.read(self.DISK_WRITE_BUFFER_SIZE) while bytes_read: temp_file.write(bytes_read) - bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE) + bytes_read = msg.read(self.DISK_WRITE_BUFFER_SIZE) CuraApplication.getInstance().processEvents() - if not CuraApplication.getInstance().getPackageManager().installPackage(temp_file.name): - redownload_errors.append(f"Couldn't install package '{package_id}'.") + temp_file.close() + if not CuraApplication.getInstance().getPackageManager().installPackage(temp_file.name): + redownload_errors.append(f"Couldn't install package '{package_id}'.") except IOError as ex: redownload_errors.append(f"Couldn't process package '{package_id}' because '{ex}'.") From 1fb89e0e7d80635e60ce4bf38e2b1853c66d4ac8 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 2 Apr 2025 13:19:01 +0200 Subject: [PATCH 8/9] Restore Backups: Correctly handle requests to redownload plugins. - Fix: Need to fill the package's API version into the (template) URL, instead of using latest. - Fix: Loop was closing over the package ID, which caused the recieving function to always only get the last handled package ID instead of the one it needed to handle. part of CURA-12156 --- plugins/CuraDrive/src/RestoreBackupJob.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/CuraDrive/src/RestoreBackupJob.py b/plugins/CuraDrive/src/RestoreBackupJob.py index aa48ca5c18..817d819abf 100644 --- a/plugins/CuraDrive/src/RestoreBackupJob.py +++ b/plugins/CuraDrive/src/RestoreBackupJob.py @@ -22,7 +22,7 @@ from cura.CuraApplication import CuraApplication from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope import cura.UltimakerCloud.UltimakerCloudConstants as UltimakerCloudConstants -PACKAGES_URL = f"{UltimakerCloudConstants.CuraCloudAPIRoot}/cura-packages/v{UltimakerCloudConstants.CuraCloudAPIVersion}/cura/v{CuraSDKVersion}/packages" +PACKAGES_URL_TEMPLATE = f"{UltimakerCloudConstants.CuraCloudAPIRoot}/cura-packages/v{UltimakerCloudConstants.CuraCloudAPIVersion}/cura/v{{0}}/packages/{{1}}/download" class RestoreBackupJob(Job): """Downloads a backup and overwrites local configuration with the backup. @@ -163,13 +163,15 @@ class RestoreBackupJob(Job): self._job_done.set() self._package_download_scope = UltimakerCloudScope(CuraApplication.getInstance()) - for package_id in to_install: - HttpRequestManager.getInstance().get( - f"{PACKAGES_URL}/{package_id}/download", - scope=self._package_download_scope, - callback=lambda msg: packageDownloadCallback(package_id, msg), - error_callback=lambda msg, err: packageDownloadCallback(package_id, msg, err) - ) + for package_id, package_api_version in to_install.items(): + def handlePackageId(package_id: str = package_id): + HttpRequestManager.getInstance().get( + PACKAGES_URL_TEMPLATE.format(package_api_version, package_id), + scope=self._package_download_scope, + callback=lambda msg: packageDownloadCallback(package_id, msg), + error_callback=lambda msg, err: packageDownloadCallback(package_id, msg, err) + ) + handlePackageId(package_id) @staticmethod def _verifyMd5Hash(file_path: str, known_hash: str) -> bool: From 14bf34d96a3fbccdc69c4c14fcef4f9486507e5d Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 15 Apr 2025 21:08:33 +0200 Subject: [PATCH 9/9] Adjust code to review comments. - Use delete-on-close instead. - Prevent infinite loops. part of CURA-12156 --- cura/Backups/Backup.py | 2 +- plugins/CuraDrive/src/CreateBackupJob.py | 6 +++--- plugins/CuraDrive/src/RestoreBackupJob.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cura/Backups/Backup.py b/cura/Backups/Backup.py index a409c4c689..1438ce293a 100644 --- a/cura/Backups/Backup.py +++ b/cura/Backups/Backup.py @@ -114,7 +114,7 @@ class Backup: data["installed"] = keep_in data["to_install"].update(reinstall) if data is not None: - tmpfile = tempfile.NamedTemporaryFile(delete=False) + tmpfile = tempfile.NamedTemporaryFile(delete_on_close=False) with open(tmpfile.name, "w") as outfile: json.dump(data, outfile) add_to_archive(tmpfile.name, file_path) diff --git a/plugins/CuraDrive/src/CreateBackupJob.py b/plugins/CuraDrive/src/CreateBackupJob.py index 67083d2e83..4820f886ab 100644 --- a/plugins/CuraDrive/src/CreateBackupJob.py +++ b/plugins/CuraDrive/src/CreateBackupJob.py @@ -99,10 +99,10 @@ class CreateBackupJob(Job): self._requestUploadSlot(backup_meta_data, len(self._backup_zip)) # Note: One 'process events' call wasn't enough with the changed situation somehow. - active_done_check = False - while not active_done_check: + for _ in range(5000): CuraApplication.getInstance().processEvents() - active_done_check = self._job_done.wait(0.02) + if self._job_done.wait(0.02): + break if self.backup_upload_error_message == "": self._upload_message.setText(catalog.i18nc("@info:backup_status", "Your backup has finished uploading.")) diff --git a/plugins/CuraDrive/src/RestoreBackupJob.py b/plugins/CuraDrive/src/RestoreBackupJob.py index 817d819abf..503b39547a 100644 --- a/plugins/CuraDrive/src/RestoreBackupJob.py +++ b/plugins/CuraDrive/src/RestoreBackupJob.py @@ -56,10 +56,10 @@ class RestoreBackupJob(Job): ) # Note: Just to be sure, use the same structure here as in CreateBackupJob. - active_done_check = False - while not active_done_check: + for _ in range(5000): CuraApplication.getInstance().processEvents() - active_done_check = self._job_done.wait(0.02) + if self._job_done.wait(0.02): + break def _onRestoreRequestCompleted(self, reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None: if not HttpRequestManager.replyIndicatesSuccess(reply, error): @@ -71,7 +71,7 @@ class RestoreBackupJob(Job): # We store the file in a temporary path fist to ensure integrity. try: - self._temporary_backup_file = NamedTemporaryFile(delete = False) + self._temporary_backup_file = NamedTemporaryFile(delete_on_close = False) with open(self._temporary_backup_file.name, "wb") as write_backup: app = CuraApplication.getInstance() bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE)