Merge pull request #7211 from Ultimaker/CURA-7150_proper_http_request_headers

CURA-7150_proper_http_request_headers

cura/API/Account.py

@@ -4,12 +4,11 @@ from typing import Optional, Dict, TYPE_CHECKING
 
 from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, pyqtProperty
 
-from UM.i18n import i18nCatalog
 from UM.Message import Message
-from cura import UltimakerCloudAuthentication
+from UM.i18n import i18nCatalog
-
 from cura.OAuth2.AuthorizationService import AuthorizationService
 from cura.OAuth2.Models import OAuth2Settings
+from cura.UltimakerCloud import UltimakerCloudAuthentication
 
 if TYPE_CHECKING:
     from cura.CuraApplication import CuraApplication

cura/CuraApplication.py

@@ -7,71 +7,52 @@ import time
 from typing import cast, TYPE_CHECKING, Optional, Callable, List, Any
 
 import numpy
-
 from PyQt5.QtCore import QObject, QTimer, QUrl, pyqtSignal, pyqtProperty, QEvent, Q_ENUMS
 from PyQt5.QtGui import QColor, QIcon
-from PyQt5.QtWidgets import QMessageBox
 from PyQt5.QtQml import qmlRegisterUncreatableType, qmlRegisterSingletonType, qmlRegisterType
+from PyQt5.QtWidgets import QMessageBox
 
-from UM.i18n import i18nCatalog
+import UM.Util
+import cura.Settings.cura_empty_instance_containers
 from UM.Application import Application
 from UM.Decorators import override
 from UM.FlameProfiler import pyqtSlot
 from UM.Logger import Logger
-from UM.Message import Message
-from UM.Platform import Platform
-from UM.PluginError import PluginNotFoundError
-from UM.Resources import Resources
-from UM.Preferences import Preferences
-from UM.Qt.QtApplication import QtApplication  # The class we're inheriting from.
-import UM.Util
-from UM.View.SelectionPass import SelectionPass  # For typing.
-
 from UM.Math.AxisAlignedBox import AxisAlignedBox
 from UM.Math.Matrix import Matrix
 from UM.Math.Quaternion import Quaternion
 from UM.Math.Vector import Vector
-
 from UM.Mesh.ReadMeshJob import ReadMeshJob
-
+from UM.Message import Message
 from UM.Operations.AddSceneNodeOperation import AddSceneNodeOperation
 from UM.Operations.GroupedOperation import GroupedOperation
 from UM.Operations.SetTransformOperation import SetTransformOperation
-
+from UM.Platform import Platform
+from UM.PluginError import PluginNotFoundError
+from UM.Preferences import Preferences
+from UM.Qt.QtApplication import QtApplication  # The class we're inheriting from.
+from UM.Resources import Resources
 from UM.Scene.Camera import Camera
 from UM.Scene.GroupDecorator import GroupDecorator
 from UM.Scene.Iterator.DepthFirstIterator import DepthFirstIterator
 from UM.Scene.SceneNode import SceneNode
 from UM.Scene.Selection import Selection
 from UM.Scene.ToolHandle import ToolHandle
-
 from UM.Settings.ContainerRegistry import ContainerRegistry
 from UM.Settings.InstanceContainer import InstanceContainer
 from UM.Settings.SettingDefinition import SettingDefinition, DefinitionPropertyType
 from UM.Settings.SettingFunction import SettingFunction
 from UM.Settings.Validator import Validator
-
+from UM.View.SelectionPass import SelectionPass  # For typing.
 from UM.Workspace.WorkspaceReader import WorkspaceReader
-
+from UM.i18n import i18nCatalog
+from cura import ApplicationMetadata
 from cura.API import CuraAPI
-
 from cura.Arranging.Arrange import Arrange
-from cura.Arranging.ArrangeObjectsJob import ArrangeObjectsJob
 from cura.Arranging.ArrangeObjectsAllBuildPlatesJob import ArrangeObjectsAllBuildPlatesJob
+from cura.Arranging.ArrangeObjectsJob import ArrangeObjectsJob
 from cura.Arranging.ShapeArray import ShapeArray
-
-from cura.Operations.SetParentOperation import SetParentOperation
-
-from cura.Scene.BlockSlicingDecorator import BlockSlicingDecorator
-from cura.Scene.BuildPlateDecorator import BuildPlateDecorator
-from cura.Scene.ConvexHullDecorator import ConvexHullDecorator
-from cura.Scene.CuraSceneController import CuraSceneController
-from cura.Scene.CuraSceneNode import CuraSceneNode
-
-from cura.Scene.SliceableObjectDecorator import SliceableObjectDecorator
-from cura.Scene import ZOffsetDecorator
 from cura.Machines.MachineErrorChecker import MachineErrorChecker
-
 from cura.Machines.Models.BuildPlateModel import BuildPlateModel
 from cura.Machines.Models.CustomQualityProfilesDropDownMenuModel import CustomQualityProfilesDropDownMenuModel
 from cura.Machines.Models.DiscoveredPrintersModel import DiscoveredPrintersModel
@@ -80,6 +61,8 @@ from cura.Machines.Models.FavoriteMaterialsModel import FavoriteMaterialsModel
 from cura.Machines.Models.FirstStartMachineActionsModel import FirstStartMachineActionsModel
 from cura.Machines.Models.GenericMaterialsModel import GenericMaterialsModel
 from cura.Machines.Models.GlobalStacksModel import GlobalStacksModel
+from cura.Machines.Models.IntentCategoryModel import IntentCategoryModel
+from cura.Machines.Models.IntentModel import IntentModel
 from cura.Machines.Models.MaterialBrandsModel import MaterialBrandsModel
 from cura.Machines.Models.MaterialManagementModel import MaterialManagementModel
 from cura.Machines.Models.MultiBuildPlateModel import MultiBuildPlateModel
@@ -89,51 +72,47 @@ from cura.Machines.Models.QualityProfilesDropDownMenuModel import QualityProfile
 from cura.Machines.Models.QualitySettingsModel import QualitySettingsModel
 from cura.Machines.Models.SettingVisibilityPresetsModel import SettingVisibilityPresetsModel
 from cura.Machines.Models.UserChangesModel import UserChangesModel
-from cura.Machines.Models.IntentModel import IntentModel
+from cura.Operations.SetParentOperation import SetParentOperation
-from cura.Machines.Models.IntentCategoryModel import IntentCategoryModel
-
-from cura.PrinterOutput.PrinterOutputDevice import PrinterOutputDevice
 from cura.PrinterOutput.NetworkMJPGImage import NetworkMJPGImage
-
+from cura.PrinterOutput.PrinterOutputDevice import PrinterOutputDevice
-import cura.Settings.cura_empty_instance_containers
+from cura.Scene import ZOffsetDecorator
+from cura.Scene.BlockSlicingDecorator import BlockSlicingDecorator
+from cura.Scene.BuildPlateDecorator import BuildPlateDecorator
+from cura.Scene.ConvexHullDecorator import ConvexHullDecorator
+from cura.Scene.CuraSceneController import CuraSceneController
+from cura.Scene.CuraSceneNode import CuraSceneNode
+from cura.Scene.SliceableObjectDecorator import SliceableObjectDecorator
 from cura.Settings.ContainerManager import ContainerManager
 from cura.Settings.CuraContainerRegistry import CuraContainerRegistry
 from cura.Settings.CuraFormulaFunctions import CuraFormulaFunctions
 from cura.Settings.ExtruderManager import ExtruderManager
 from cura.Settings.ExtruderStack import ExtruderStack
+from cura.Settings.GlobalStack import GlobalStack
+from cura.Settings.IntentManager import IntentManager
 from cura.Settings.MachineManager import MachineManager
 from cura.Settings.MachineNameValidator import MachineNameValidator
-from cura.Settings.IntentManager import IntentManager
 from cura.Settings.MaterialSettingsVisibilityHandler import MaterialSettingsVisibilityHandler
 from cura.Settings.SettingInheritanceManager import SettingInheritanceManager
 from cura.Settings.SidebarCustomMenuItemsModel import SidebarCustomMenuItemsModel
 from cura.Settings.SimpleModeSettingsManager import SimpleModeSettingsManager
-
 from cura.TaskManagement.OnExitCallbackManager import OnExitCallbackManager
-
 from cura.UI import CuraSplashScreen, MachineActionManager, PrintInformation
+from cura.UI.AddPrinterPagesModel import AddPrinterPagesModel
 from cura.UI.MachineSettingsManager import MachineSettingsManager
 from cura.UI.ObjectsModel import ObjectsModel
-from cura.UI.TextManager import TextManager
-from cura.UI.AddPrinterPagesModel import AddPrinterPagesModel
 from cura.UI.RecommendedMode import RecommendedMode
+from cura.UI.TextManager import TextManager
 from cura.UI.WelcomePagesModel import WelcomePagesModel
 from cura.UI.WhatsNewPagesModel import WhatsNewPagesModel
-
+from cura.UltimakerCloud import UltimakerCloudAuthentication
 from cura.Utils.NetworkingUtil import NetworkingUtil
-
-from .SingleInstance import SingleInstance
-from .AutoSave import AutoSave
-from . import PlatformPhysics
 from . import BuildVolume
 from . import CameraAnimation
 from . import CuraActions
+from . import PlatformPhysics
 from . import PrintJobPreviewImageProvider
-
+from .AutoSave import AutoSave
-from cura.TaskManagement.OnExitCallbackManager import OnExitCallbackManager
+from .SingleInstance import SingleInstance
-
-from cura import ApplicationMetadata, UltimakerCloudAuthentication
-from cura.Settings.GlobalStack import GlobalStack
 
 if TYPE_CHECKING:
     from UM.Settings.EmptyInstanceContainer import EmptyInstanceContainer

cura/UltimakerCloud/UltimakerCloudScope.py

@@ -6,17 +6,20 @@ from cura.API import Account
 from cura.CuraApplication import CuraApplication
 
 
-## Add a Authorization header to the request for Ultimaker Cloud Api requests.
-# When the user is not logged in or a token is not available, a warning will be logged
-# Also add the user agent headers (see DefaultUserAgentScope)
 class UltimakerCloudScope(DefaultUserAgentScope):
+    """Add an Authorization header to the request for Ultimaker Cloud Api requests.
+
+    When the user is not logged in or a token is not available, a warning will be logged
+    Also add the user agent headers (see DefaultUserAgentScope)
+    """
+
     def __init__(self, application: CuraApplication):
         super().__init__(application)
         api = application.getCuraAPI()
         self._account = api.account  # type: Account
 
-    def request_hook(self, request: QNetworkRequest):
+    def requestHook(self, request: QNetworkRequest):
-        super().request_hook(request)
+        super().requestHook(request)
         token = self._account.accessToken
         if not self._account.isLoggedIn or token is None:
             Logger.warning("Cannot add authorization to Cloud Api request")
@@ -25,4 +28,4 @@ class UltimakerCloudScope(DefaultUserAgentScope):
         header_dict = {
             "Authorization": "Bearer {}".format(token)
         }
-        self.add_headers(request, header_dict)
+        self.addHeaders(request, header_dict)

cura_app.py

@@ -23,6 +23,8 @@ import os
 import Arcus  # @UnusedImport
 import Savitar  # @UnusedImport
 
+from PyQt5.QtNetwork import QSslConfiguration, QSslSocket
+
 from UM.Platform import Platform
 from cura import ApplicationMetadata
 from cura.ApplicationMetadata import CuraAppName
@@ -220,5 +222,10 @@ if Platform.isLinux() and getattr(sys, "frozen", False):
     import trimesh.exchange.load
     os.environ["LD_LIBRARY_PATH"] = old_env
 
+if ApplicationMetadata.CuraDebugMode:
+    ssl_conf = QSslConfiguration.defaultConfiguration()
+    ssl_conf.setPeerVerifyMode(QSslSocket.VerifyNone)
+    QSslConfiguration.setDefaultConfiguration(ssl_conf)
+
 app = CuraApplication()
 app.run()

plugins/CuraDrive/src/CreateBackupJob.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2020 Ultimaker B.V.
+# Cura is released under the terms of the LGPLv3 or higher.
+import json
+import threading
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from PyQt5.QtNetwork import QNetworkReply, QNetworkRequest
+
+from UM.Job import Job
+from UM.Logger import Logger
+from UM.Message import Message
+from UM.TaskManagement.HttpRequestManager import HttpRequestManager
+from UM.TaskManagement.HttpRequestScope import JsonDecoratorScope
+from UM.i18n import i18nCatalog
+from cura.CuraApplication import CuraApplication
+from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope
+
+catalog = i18nCatalog("cura")
+
+
+class CreateBackupJob(Job):
+    """Creates backup zip, requests upload url and uploads the backup file to cloud storage."""
+
+    MESSAGE_TITLE = catalog.i18nc("@info:title", "Backups")
+    DEFAULT_UPLOAD_ERROR_MESSAGE = catalog.i18nc("@info:backup_status", "There was an error while uploading your backup.")
+
+    def __init__(self, api_backup_url: str) -> None:
+        """ Create a new backup Job. start the job by calling start()
+
+        :param api_backup_url: The url of the 'backups' endpoint of the Cura Drive Api
+        """
+
+        super().__init__()
+
+        self._api_backup_url = api_backup_url
+        self._json_cloud_scope = JsonDecoratorScope(UltimakerCloudScope(CuraApplication.getInstance()))
+
+        self._backup_zip = None  # type: Optional[bytes]
+        self._job_done = threading.Event()
+        """Set when the job completes. Does not indicate success."""
+        self.backup_upload_error_message = ""
+        """After the job completes, an empty string indicates success. Othrerwise, the value is a translated message."""
+
+    def run(self) -> None:
+        upload_message = Message(catalog.i18nc("@info:backup_status", "Creating your backup..."), title = self.MESSAGE_TITLE, progress = -1)
+        upload_message.show()
+        CuraApplication.getInstance().processEvents()
+        cura_api = CuraApplication.getInstance().getCuraAPI()
+        self._backup_zip, backup_meta_data = cura_api.backups.createBackup()
+
+        if not self._backup_zip or not backup_meta_data:
+            self.backup_upload_error_message = catalog.i18nc("@info:backup_status", "There was an error while creating your backup.")
+            upload_message.hide()
+            return
+
+        upload_message.setText(catalog.i18nc("@info:backup_status", "Uploading your backup..."))
+        CuraApplication.getInstance().processEvents()
+
+        # Create an upload entry for the backup.
+        timestamp = datetime.now().isoformat()
+        backup_meta_data["description"] = "{}.backup.{}.cura.zip".format(timestamp, backup_meta_data["cura_release"])
+        self._requestUploadSlot(backup_meta_data, len(self._backup_zip))
+
+        self._job_done.wait()
+        if self.backup_upload_error_message == "":
+            upload_message.setText(catalog.i18nc("@info:backup_status", "Your backup has finished uploading."))
+            upload_message.setProgress(None)  # Hide progress bar
+        else:
+            # some error occurred. This error is presented to the user by DrivePluginExtension
+            upload_message.hide()
+
+    def _requestUploadSlot(self, backup_metadata: Dict[str, Any], backup_size: int) -> None:
+        """Request a backup upload slot from the API.
+
+        :param backup_metadata: A dict containing some meta data about the backup.
+        :param backup_size: The size of the backup file in bytes.
+        """
+
+        payload = json.dumps({"data": {"backup_size": backup_size,
+                                       "metadata": backup_metadata
+                                       }
+                              }).encode()
+
+        HttpRequestManager.getInstance().put(
+            self._api_backup_url,
+            data = payload,
+            callback = self._onUploadSlotCompleted,
+            error_callback = self._onUploadSlotCompleted,
+            scope = self._json_cloud_scope)
+
+    def _onUploadSlotCompleted(self, reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None:
+        if error is not None:
+            Logger.warning(str(error))
+            self.backup_upload_error_message = self.DEFAULT_UPLOAD_ERROR_MESSAGE
+            self._job_done.set()
+            return
+        if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) >= 300:
+            Logger.warning("Could not request backup upload: %s", HttpRequestManager.readText(reply))
+            self.backup_upload_error_message = self.DEFAULT_UPLOAD_ERROR_MESSAGE
+            self._job_done.set()
+            return
+
+        backup_upload_url = HttpRequestManager.readJSON(reply)["data"]["upload_url"]
+
+        # Upload the backup to storage.
+        HttpRequestManager.getInstance().put(
+            backup_upload_url,
+            data=self._backup_zip,
+            callback=self._uploadFinishedCallback,
+            error_callback=self._uploadFinishedCallback
+        )
+
+    def _uploadFinishedCallback(self, reply: QNetworkReply, error: QNetworkReply.NetworkError = None):
+        if not HttpRequestManager.replyIndicatesSuccess(reply, error):
+            Logger.log("w", "Could not upload backup file: %s", HttpRequestManager.readText(reply))
+            self.backup_upload_error_message = self.DEFAULT_UPLOAD_ERROR_MESSAGE
+
+        self._job_done.set()

plugins/CuraDrive/src/DriveApiService.py

@@ -1,90 +1,70 @@
 # Copyright (c) 2018 Ultimaker B.V.
 # Cura is released under the terms of the LGPLv3 or higher.
 
-import base64
+from typing import Any, Optional, List, Dict, Callable
-import hashlib
-from datetime import datetime
-from tempfile import NamedTemporaryFile
-from typing import Any, Optional, List, Dict
 
-import requests
+from PyQt5.QtNetwork import QNetworkReply
 
 from UM.Logger import Logger
-from UM.Message import Message
 from UM.Signal import Signal, signalemitter
+from UM.TaskManagement.HttpRequestManager import HttpRequestManager
+from UM.TaskManagement.HttpRequestScope import JsonDecoratorScope
+from UM.i18n import i18nCatalog
 from cura.CuraApplication import CuraApplication
-
+from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope
-from .UploadBackupJob import UploadBackupJob
+from .CreateBackupJob import CreateBackupJob
+from .RestoreBackupJob import RestoreBackupJob
 from .Settings import Settings
 
-from UM.i18n import i18nCatalog
 catalog = i18nCatalog("cura")
 
 
-## The DriveApiService is responsible for interacting with the CuraDrive API and Cura's backup handling.
 @signalemitter
 class DriveApiService:
+    """The DriveApiService is responsible for interacting with the CuraDrive API and Cura's backup handling."""
+
     BACKUP_URL = "{}/backups".format(Settings.DRIVE_API_URL)
 
-    # Emit signal when restoring backup started or finished.
     restoringStateChanged = Signal()
+    """Emits signal when restoring backup started or finished."""
 
-    # Emit signal when creating backup started or finished.
     creatingStateChanged = Signal()
+    """Emits signal when creating backup started or finished."""
 
     def __init__(self) -> None:
         self._cura_api = CuraApplication.getInstance().getCuraAPI()
+        self._json_cloud_scope = JsonDecoratorScope(UltimakerCloudScope(CuraApplication.getInstance()))
 
-    def getBackups(self) -> List[Dict[str, Any]]:
+    def getBackups(self, changed: Callable[[List[Dict[str, Any]]], None]) -> None:
-        access_token = self._cura_api.account.accessToken
+        def callback(reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None:
-        if not access_token:
+            if error is not None:
-            Logger.log("w", "Could not get access token.")
+                Logger.log("w", "Could not get backups: " + str(error))
-            return []
+                changed([])
-        try:
+                return
-            backup_list_request = requests.get(self.BACKUP_URL, headers = {
-                "Authorization": "Bearer {}".format(access_token)
-            })
-        except requests.exceptions.ConnectionError:
-            Logger.logException("w", "Unable to connect with the server.")
-            return []
 
-        # HTTP status 300s mean redirection. 400s and 500s are errors.
+            backup_list_response = HttpRequestManager.readJSON(reply)
-        # Technically 300s are not errors, but the use case here relies on "requests" to handle redirects automatically.
+            if "data" not in backup_list_response:
-        if backup_list_request.status_code >= 300:
+                Logger.log("w", "Could not get backups from remote, actual response body was: %s",
-            Logger.log("w", "Could not get backups list from remote: %s", backup_list_request.text)
+                           str(backup_list_response))
-            Message(catalog.i18nc("@info:backup_status", "There was an error listing your backups."), title = catalog.i18nc("@info:title", "Backup")).show()
+                changed([])  # empty list of backups
-            return []
+                return
 
-        backup_list_response = backup_list_request.json()
+            changed(backup_list_response["data"])
-        if "data" not in backup_list_response:
-            Logger.log("w", "Could not get backups from remote, actual response body was: %s", str(backup_list_response))
-            return []
 
-        return backup_list_response["data"]
+        HttpRequestManager.getInstance().get(
+            self.BACKUP_URL,
+            callback= callback,
+            error_callback = callback,
+            scope=self._json_cloud_scope
+        )
 
     def createBackup(self) -> None:
         self.creatingStateChanged.emit(is_creating = True)
-
+        upload_backup_job = CreateBackupJob(self.BACKUP_URL)
-        # Create the backup.
-        backup_zip_file, backup_meta_data = self._cura_api.backups.createBackup()
-        if not backup_zip_file or not backup_meta_data:
-            self.creatingStateChanged.emit(is_creating = False, error_message ="Could not create backup.")
-            return
-
-        # Create an upload entry for the backup.
-        timestamp = datetime.now().isoformat()
-        backup_meta_data["description"] = "{}.backup.{}.cura.zip".format(timestamp, backup_meta_data["cura_release"])
-        backup_upload_url = self._requestBackupUpload(backup_meta_data, len(backup_zip_file))
-        if not backup_upload_url:
-            self.creatingStateChanged.emit(is_creating = False, error_message ="Could not upload backup.")
-            return
-
-        # Upload the backup to storage.
-        upload_backup_job = UploadBackupJob(backup_upload_url, backup_zip_file)
         upload_backup_job.finished.connect(self._onUploadFinished)
         upload_backup_job.start()
 
-    def _onUploadFinished(self, job: "UploadBackupJob") -> None:
+    def _onUploadFinished(self, job: "CreateBackupJob") -> None:
         if job.backup_upload_error_message != "":
             # If the job contains an error message we pass it along so the UI can display it.
             self.creatingStateChanged.emit(is_creating = False, error_message = job.backup_upload_error_message)
@@ -96,96 +76,38 @@ class DriveApiService:
         download_url = backup.get("download_url")
         if not download_url:
             # If there is no download URL, we can't restore the backup.
-            return self._emitRestoreError()
+            Logger.warning("backup download_url is missing. Aborting backup.")
+            self.restoringStateChanged.emit(is_restoring = False,
+                                            error_message = catalog.i18nc("@info:backup_status",
+                                                                        "There was an error trying to restore your backup."))
+            return
 
-        try:
+        restore_backup_job = RestoreBackupJob(backup)
-            download_package = requests.get(download_url, stream = True)
+        restore_backup_job.finished.connect(self._onRestoreFinished)
-        except requests.exceptions.ConnectionError:
+        restore_backup_job.start()
-            Logger.logException("e", "Unable to connect with the server")
-            return self._emitRestoreError()
 
-        if download_package.status_code >= 300:
+    def _onRestoreFinished(self, job: "RestoreBackupJob") -> None:
-            # Something went wrong when attempting to download the backup.
+        if job.restore_backup_error_message != "":
-            Logger.log("w", "Could not download backup from url %s: %s", download_url, download_package.text)
+            # If the job contains an error message we pass it along so the UI can display it.
-            return self._emitRestoreError()
+            self.restoringStateChanged.emit(is_restoring=False)
+        else:
+            self.restoringStateChanged.emit(is_restoring = False, error_message = job.restore_backup_error_message)
 
-        # We store the file in a temporary path fist to ensure integrity.
+    def deleteBackup(self, backup_id: str, finished_callable: Callable[[bool], None]):
-        temporary_backup_file = NamedTemporaryFile(delete = False)
-        with open(temporary_backup_file.name, "wb") as write_backup:
-            for chunk in download_package:
-                write_backup.write(chunk)
 
-        if not self._verifyMd5Hash(temporary_backup_file.name, backup.get("md5_hash", "")):
+        def finishedCallback(reply: QNetworkReply, ca: Callable[[bool], None] = finished_callable) -> None:
-            # Don't restore the backup if the MD5 hashes do not match.
+            self._onDeleteRequestCompleted(reply, ca)
-            # This can happen if the download was interrupted.
-            Logger.log("w", "Remote and local MD5 hashes do not match, not restoring backup.")
-            return self._emitRestoreError()
 
-        # Tell Cura to place the backup back in the user data folder.
+        def errorCallback(reply: QNetworkReply, error: QNetworkReply.NetworkError, ca: Callable[[bool], None] = finished_callable) -> None:
-        with open(temporary_backup_file.name, "rb") as read_backup:
+            self._onDeleteRequestCompleted(reply, ca, error)
-            self._cura_api.backups.restoreBackup(read_backup.read(), backup.get("metadata", {}))
-            self.restoringStateChanged.emit(is_restoring = False)
 
-    def _emitRestoreError(self) -> None:
+        HttpRequestManager.getInstance().delete(
-        self.restoringStateChanged.emit(is_restoring = False,
+            url = "{}/{}".format(self.BACKUP_URL, backup_id),
-                                        error_message = catalog.i18nc("@info:backup_status",
+            callback = finishedCallback,
-                                                                         "There was an error trying to restore your backup."))
+            error_callback = errorCallback,
+            scope= self._json_cloud_scope
+        )
 
-    #   Verify the MD5 hash of a file.
-    #   \param file_path Full path to the file.
-    #   \param known_hash The known MD5 hash of the file.
-    #   \return: Success or not.
     @staticmethod
-    def _verifyMd5Hash(file_path: str, known_hash: str) -> bool:
+    def _onDeleteRequestCompleted(reply: QNetworkReply, callable: Callable[[bool], None], error: Optional["QNetworkReply.NetworkError"] = None) -> None:
-        with open(file_path, "rb") as read_backup:
+        callable(HttpRequestManager.replyIndicatesSuccess(reply, error))
-            local_md5_hash = base64.b64encode(hashlib.md5(read_backup.read()).digest(), altchars = b"_-").decode("utf-8")
-            return known_hash == local_md5_hash
-
-    def deleteBackup(self, backup_id: str) -> bool:
-        access_token = self._cura_api.account.accessToken
-        if not access_token:
-            Logger.log("w", "Could not get access token.")
-            return False
-
-        try:
-            delete_backup = requests.delete("{}/{}".format(self.BACKUP_URL, backup_id), headers = {
-                "Authorization": "Bearer {}".format(access_token)
-            })
-        except requests.exceptions.ConnectionError:
-            Logger.logException("e", "Unable to connect with the server")
-            return False
-
-        if delete_backup.status_code >= 300:
-            Logger.log("w", "Could not delete backup: %s", delete_backup.text)
-            return False
-        return True
-
-    #   Request a backup upload slot from the API.
-    #   \param backup_metadata: A dict containing some meta data about the backup.
-    #   \param backup_size The size of the backup file in bytes.
-    #   \return: The upload URL for the actual backup file if successful, otherwise None.
-    def _requestBackupUpload(self, backup_metadata: Dict[str, Any], backup_size: int) -> Optional[str]:
-        access_token = self._cura_api.account.accessToken
-        if not access_token:
-            Logger.log("w", "Could not get access token.")
-            return None
-        try:
-            backup_upload_request = requests.put(
-                self.BACKUP_URL,
-                json = {"data": {"backup_size": backup_size,
-                                 "metadata": backup_metadata
-                                 }
-                        },
-                headers = {
-                    "Authorization": "Bearer {}".format(access_token)
-                })
-        except requests.exceptions.ConnectionError:
-            Logger.logException("e", "Unable to connect with the server")
-            return None
-
-        # Any status code of 300 or above indicates an error.
-        if backup_upload_request.status_code >= 300:
-            Logger.log("w", "Could not request backup upload: %s", backup_upload_request.text)
-            return None
-        
-        return backup_upload_request.json()["data"]["upload_url"]

plugins/CuraDrive/src/DrivePluginExtension.py

@@ -133,7 +133,10 @@ class DrivePluginExtension(QObject, Extension):
 
     @pyqtSlot(name = "refreshBackups")
     def refreshBackups(self) -> None:
-        self._backups = self._drive_api_service.getBackups()
+        self._drive_api_service.getBackups(self._backupsChangedCallback)
+
+    def _backupsChangedCallback(self, backups: List[Dict[str, Any]]) -> None:
+        self._backups = backups
         self.backupsChanged.emit()
 
     @pyqtProperty(bool, notify = restoringStateChanged)
@@ -158,5 +161,8 @@ class DrivePluginExtension(QObject, Extension):
 
     @pyqtSlot(str, name = "deleteBackup")
     def deleteBackup(self, backup_id: str) -> None:
-        self._drive_api_service.deleteBackup(backup_id)
+        self._drive_api_service.deleteBackup(backup_id, self._backupDeletedCallback)
-        self.refreshBackups()
+
+    def _backupDeletedCallback(self, success: bool):
+        if success:
+            self.refreshBackups()

plugins/CuraDrive/src/RestoreBackupJob.py

@@ -0,0 +1,92 @@
+import base64
+import hashlib
+import threading
+from tempfile import NamedTemporaryFile
+from typing import Optional, Any, Dict
+
+from PyQt5.QtNetwork import QNetworkReply, QNetworkRequest
+
+from UM.Job import Job
+from UM.Logger import Logger
+from UM.PackageManager import catalog
+from UM.TaskManagement.HttpRequestManager import HttpRequestManager
+from cura.CuraApplication import CuraApplication
+
+
+class RestoreBackupJob(Job):
+    """Downloads a backup and overwrites local configuration with the backup.
+
+     When `Job.finished` emits, `restore_backup_error_message` will either be `""` (no error) or an error message
+     """
+
+    DISK_WRITE_BUFFER_SIZE = 512 * 1024
+    DEFAULT_ERROR_MESSAGE = catalog.i18nc("@info:backup_status", "There was an error trying to restore your backup.")
+
+    def __init__(self, backup: Dict[str, Any]) -> None:
+        """ Create a new restore Job. start the job by calling start()
+
+        :param backup: A dict containing a backup spec
+        """
+
+        super().__init__()
+        self._job_done = threading.Event()
+
+        self._backup = backup
+        self.restore_backup_error_message = ""
+
+    def run(self) -> None:
+
+        url = self._backup.get("download_url")
+        assert url is not None
+
+        HttpRequestManager.getInstance().get(
+            url = url,
+            callback = self._onRestoreRequestCompleted,
+            error_callback = self._onRestoreRequestCompleted
+        )
+
+        self._job_done.wait()  # A job is considered finished when the run function completes
+
+    def _onRestoreRequestCompleted(self, reply: QNetworkReply, error: Optional["QNetworkReply.NetworkError"] = None) -> None:
+        if not HttpRequestManager.replyIndicatesSuccess(reply, error):
+            Logger.warning("Requesting backup failed, response code %s while trying to connect to %s",
+                           reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url())
+            self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE
+            self._job_done.set()
+            return
+
+        # We store the file in a temporary path fist to ensure integrity.
+        temporary_backup_file = NamedTemporaryFile(delete = False)
+        with open(temporary_backup_file.name, "wb") as write_backup:
+            app = CuraApplication.getInstance()
+            bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE)
+            while bytes_read:
+                write_backup.write(bytes_read)
+                bytes_read = reply.read(self.DISK_WRITE_BUFFER_SIZE)
+                app.processEvents()
+
+        if not self._verifyMd5Hash(temporary_backup_file.name, self._backup.get("md5_hash", "")):
+            # Don't restore the backup if the MD5 hashes do not match.
+            # This can happen if the download was interrupted.
+            Logger.log("w", "Remote and local MD5 hashes do not match, not restoring backup.")
+            self.restore_backup_error_message = self.DEFAULT_ERROR_MESSAGE
+
+        # Tell Cura to place the backup back in the user data folder.
+        with open(temporary_backup_file.name, "rb") as read_backup:
+            cura_api = CuraApplication.getInstance().getCuraAPI()
+            cura_api.backups.restoreBackup(read_backup.read(), self._backup.get("metadata", {}))
+
+        self._job_done.set()
+
+    @staticmethod
+    def _verifyMd5Hash(file_path: str, known_hash: str) -> bool:
+        """Verify the MD5 hash of a file.
+
+        :param file_path: Full path to the file.
+        :param known_hash: The known MD5 hash of the file.
+        :return: Success or not.
+        """
+
+        with open(file_path, "rb") as read_backup:
+            local_md5_hash = base64.b64encode(hashlib.md5(read_backup.read()).digest(), altchars = b"_-").decode("utf-8")
+            return known_hash == local_md5_hash

plugins/CuraDrive/src/Settings.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2018 Ultimaker B.V.
 # Cura is released under the terms of the LGPLv3 or higher.
 
-from cura import UltimakerCloudAuthentication
+from cura.UltimakerCloud import UltimakerCloudAuthentication
 
 
 class Settings:

plugins/CuraDrive/src/UploadBackupJob.py

@@ -1,41 +0,0 @@
-# Copyright (c) 2018 Ultimaker B.V.
-# Cura is released under the terms of the LGPLv3 or higher.
-
-import requests
-
-from UM.Job import Job
-from UM.Logger import Logger
-from UM.Message import Message
-
-from UM.i18n import i18nCatalog
-catalog = i18nCatalog("cura")
-
-
-class UploadBackupJob(Job):
-    MESSAGE_TITLE = catalog.i18nc("@info:title", "Backups")
-
-    # This job is responsible for uploading the backup file to cloud storage.
-    # As it can take longer than some other tasks, we schedule this using a Cura Job.
-    def __init__(self, signed_upload_url: str, backup_zip: bytes) -> None:
-        super().__init__()
-        self._signed_upload_url = signed_upload_url
-        self._backup_zip = backup_zip
-        self._upload_success = False
-        self.backup_upload_error_message = ""
-
-    def run(self) -> None:
-        upload_message = Message(catalog.i18nc("@info:backup_status", "Uploading your backup..."), title = self.MESSAGE_TITLE, progress = -1)
-        upload_message.show()
-
-        backup_upload = requests.put(self._signed_upload_url, data = self._backup_zip)
-        upload_message.hide()
-
-        if backup_upload.status_code >= 300:
-            self.backup_upload_error_message = backup_upload.text
-            Logger.log("w", "Could not upload backup file: %s", backup_upload.text)
-            Message(catalog.i18nc("@info:backup_status", "There was an error while uploading your backup."), title = self.MESSAGE_TITLE).show()
-        else:
-            self._upload_success = True
-            Message(catalog.i18nc("@info:backup_status", "Your backup has finished uploading."), title = self.MESSAGE_TITLE).show()
-
-        self.finished.emit(self)

plugins/Toolbox/src/CloudApiModel.py

@@ -1,6 +1,7 @@
 from typing import Union
 
-from cura import ApplicationMetadata, UltimakerCloudAuthentication
+from cura import ApplicationMetadata
+from cura.UltimakerCloud import UltimakerCloudAuthentication
 
 
 class CloudApiModel:

plugins/Toolbox/src/CloudSync/CloudApiClient.py

@@ -1,8 +1,9 @@
 from UM.Logger import Logger
 from UM.TaskManagement.HttpRequestManager import HttpRequestManager
+from UM.TaskManagement.HttpRequestScope import JsonDecoratorScope
 from cura.CuraApplication import CuraApplication
+from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope
 from ..CloudApiModel import CloudApiModel
-from ..UltimakerCloudScope import UltimakerCloudScope
 
 
 class CloudApiClient:
@@ -26,7 +27,7 @@ class CloudApiClient:
         if self.__instance is not None:
             raise RuntimeError("This is a Singleton. use getInstance()")
 
-        self._scope = UltimakerCloudScope(app)  # type: UltimakerCloudScope
+        self._scope = JsonDecoratorScope(UltimakerCloudScope(app))  # type: JsonDecoratorScope
 
         app.getPackageManager().packageInstalled.connect(self._onPackageInstalled)
 

plugins/Toolbox/src/CloudSync/CloudPackageChecker.py

@@ -2,6 +2,7 @@
 # Cura is released under the terms of the LGPLv3 or higher.
 
 import json
+from typing import List, Dict, Any
 from typing import Optional
 
 from PyQt5.QtCore import QObject
@@ -11,12 +12,12 @@ from UM import i18nCatalog
 from UM.Logger import Logger
 from UM.Message import Message
 from UM.Signal import Signal
+from UM.TaskManagement.HttpRequestScope import JsonDecoratorScope
 from cura.CuraApplication import CuraApplication, ApplicationMetadata
-from ..CloudApiModel import CloudApiModel
+from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope
 from .SubscribedPackagesModel import SubscribedPackagesModel
-from ..UltimakerCloudScope import UltimakerCloudScope
+from ..CloudApiModel import CloudApiModel
 
-from typing import List, Dict, Any
 
 class CloudPackageChecker(QObject):
     def __init__(self, application: CuraApplication) -> None:
@@ -24,7 +25,7 @@ class CloudPackageChecker(QObject):
 
         self.discrepancies = Signal()  # Emits SubscribedPackagesModel
         self._application = application  # type: CuraApplication
-        self._scope = UltimakerCloudScope(application)
+        self._scope = JsonDecoratorScope(UltimakerCloudScope(application))
         self._model = SubscribedPackagesModel()
         self._message = None  # type: Optional[Message]
 
@@ -111,4 +112,4 @@ class CloudPackageChecker(QObject):
 
     def _onSyncButtonClicked(self, sync_message: Message, sync_message_action: str) -> None:
         sync_message.hide()
-        self.discrepancies.emit(self._model)
+        self.discrepancies.emit(self._model)

plugins/Toolbox/src/CloudSync/DownloadPresenter.py

@@ -12,8 +12,8 @@ from UM.Message import Message
 from UM.Signal import Signal
 from UM.TaskManagement.HttpRequestManager import HttpRequestManager
 from cura.CuraApplication import CuraApplication
+from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope
 from .SubscribedPackagesModel import SubscribedPackagesModel
-from ..UltimakerCloudScope import UltimakerCloudScope
 
 
 ## Downloads a set of packages from the Ultimaker Cloud Marketplace

plugins/Toolbox/src/Toolbox.py

@@ -9,22 +9,20 @@ from typing import cast, Any, Dict, List, Set, TYPE_CHECKING, Tuple, Optional, U
 from PyQt5.QtCore import QObject, pyqtProperty, pyqtSignal, pyqtSlot
 from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest, QNetworkReply
 
+from UM.Extension import Extension
 from UM.Logger import Logger
 from UM.PluginRegistry import PluginRegistry
-from UM.Extension import Extension
+from UM.TaskManagement.HttpRequestScope import JsonDecoratorScope
-from UM.i18n import i18nCatalog
 from UM.Version import Version
-
+from UM.i18n import i18nCatalog
 from cura import ApplicationMetadata
-
 from cura.CuraApplication import CuraApplication
 from cura.Machines.ContainerTree import ContainerTree
-
+from cura.UltimakerCloud.UltimakerCloudScope import UltimakerCloudScope
-from .CloudApiModel import CloudApiModel
 from .AuthorsModel import AuthorsModel
+from .CloudApiModel import CloudApiModel
 from .CloudSync.LicenseModel import LicenseModel
 from .PackagesModel import PackagesModel
-from .UltimakerCloudScope import UltimakerCloudScope
 
 if TYPE_CHECKING:
     from UM.TaskManagement.HttpRequestData import HttpRequestData
@@ -54,7 +52,8 @@ class Toolbox(QObject, Extension):
         self._download_request_data = None  # type: Optional[HttpRequestData]
         self._download_progress = 0  # type: float
         self._is_downloading = False  # type: bool
-        self._scope = UltimakerCloudScope(application)  # type: UltimakerCloudScope
+        self._cloud_scope = UltimakerCloudScope(application)  # type: UltimakerCloudScope
+        self._json_scope = JsonDecoratorScope(self._cloud_scope)  # type: JsonDecoratorScope
 
         self._request_urls = {}  # type: Dict[str, str]
         self._to_update = []  # type: List[str] # Package_ids that are waiting to be updated
@@ -151,7 +150,7 @@ class Toolbox(QObject, Extension):
         url = "{base_url}/packages/{package_id}/ratings".format(base_url = CloudApiModel.api_url, package_id = package_id)
         data = "{\"data\": {\"cura_version\": \"%s\", \"rating\": %i}}" % (Version(self._application.getVersion()), rating)
 
-        self._application.getHttpRequestManager().put(url, data = data.encode(), scope = self._scope)
+        self._application.getHttpRequestManager().put(url, data = data.encode(), scope = self._json_scope)
 
     def getLicenseDialogPluginFileLocation(self) -> str:
         return self._license_dialog_plugin_file_location
@@ -541,7 +540,7 @@ class Toolbox(QObject, Extension):
         self._application.getHttpRequestManager().get(url,
                                                       callback = callback,
                                                       error_callback = error_callback,
-                                                      scope=self._scope)
+                                                      scope=self._json_scope)
 
     @pyqtSlot(str)
     def startDownload(self, url: str) -> None:
@@ -554,7 +553,7 @@ class Toolbox(QObject, Extension):
                                                                      callback = callback,
                                                                      error_callback = error_callback,
                                                                      download_progress_callback = download_progress_callback,
-                                                                     scope=self._scope
+                                                                     scope=self._cloud_scope
                                                                      )
 
         self._download_request_data = request_data

plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py

@@ -9,18 +9,16 @@ from PyQt5.QtCore import QUrl
 from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManager
 
 from UM.Logger import Logger
-from cura import UltimakerCloudAuthentication
 from cura.API import Account
-
+from cura.UltimakerCloud import UltimakerCloudAuthentication
 from .ToolPathUploader import ToolPathUploader
 from ..Models.BaseModel import BaseModel
 from ..Models.Http.CloudClusterResponse import CloudClusterResponse
-from ..Models.Http.CloudError import CloudError
 from ..Models.Http.CloudClusterStatus import CloudClusterStatus
+from ..Models.Http.CloudError import CloudError
+from ..Models.Http.CloudPrintJobResponse import CloudPrintJobResponse
 from ..Models.Http.CloudPrintJobUploadRequest import CloudPrintJobUploadRequest
 from ..Models.Http.CloudPrintResponse import CloudPrintResponse
-from ..Models.Http.CloudPrintJobResponse import CloudPrintJobResponse
-
 
 ## The generic type variable used to document the methods below.
 CloudApiClientModel = TypeVar("CloudApiClientModel", bound=BaseModel)