From fed779d0d2cfd6a81fa5747a0a10debbcdb8c8ec Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Wed, 12 Dec 2018 17:31:08 +0100 Subject: [PATCH 01/16] STAR-322: Implementing multi-part upload (doesnt always work) --- .../src/Cloud/CloudApiClient.py | 23 +---- .../src/Cloud/CloudOutputDevice.py | 56 +++-------- .../src/Cloud/CloudProgressMessage.py | 37 ++++++++ .../src/Cloud/ResumableUpload.py | 94 +++++++++++++++++++ .../tests/Cloud/NetworkManagerMock.py | 3 +- .../tests/Cloud/TestCloudApiClient.py | 15 +-- 6 files changed, 161 insertions(+), 67 deletions(-) create mode 100644 plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py create mode 100644 plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index b08bac6670..2637f17010 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -9,6 +9,7 @@ from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply from UM.Logger import Logger from cura.API import Account from cura.NetworkClient import NetworkClient +from .ResumableUpload import ResumableUpload from ..Models import BaseModel from .Models.CloudClusterResponse import CloudClusterResponse from .Models.CloudErrorObject import CloudErrorObject @@ -69,24 +70,10 @@ class CloudApiClient(NetworkClient): # \param on_finished: The function to be called after the result is parsed. It receives the print job ID. # \param on_progress: A function to be called during upload progress. It receives a percentage (0-100). # \param on_error: A function to be called if the upload fails. It receives a dict with the error. - def uploadMesh(self, upload_response: CloudPrintJobResponse, mesh: bytes, on_finished: Callable[[str], Any], - on_progress: Callable[[int], Any], on_error: Callable[[dict], Any]): - - def progressCallback(bytes_sent: int, bytes_total: int) -> None: - if bytes_total: - on_progress(int((bytes_sent / bytes_total) * 100)) - - def finishedCallback(reply: QNetworkReply): - status_code, response = self._parseReply(reply) - if status_code < 300: - on_finished(upload_response.job_id) - else: - Logger.log("e", "Received unexpected response %s uploading mesh: %s", status_code, response) - on_error(response) - - # TODO: Multipart upload - self.put(upload_response.upload_url, data = mesh, content_type = upload_response.content_type, - on_finished = finishedCallback, on_progress = progressCallback) + def uploadMesh(self, upload_response: CloudPrintJobResponse, mesh: bytes, on_finished: Callable[[], Any], + on_progress: Callable[[int], Any], on_error: Callable[[], Any]): + ResumableUpload(upload_response.upload_url, upload_response.content_type, mesh, on_finished, + on_progress, on_error).start() # Requests a cluster to print the given print job. # \param cluster_id: The ID of the cluster. diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index c4ab752163..e75989a6a8 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -18,6 +18,7 @@ from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from plugins.UM3NetworkPrinting.src.Cloud.CloudOutputController import CloudOutputController from ..MeshFormatHandler import MeshFormatHandler from ..UM3PrintJobOutputModel import UM3PrintJobOutputModel +from .CloudProgressMessage import CloudProgressMessage from .CloudApiClient import CloudApiClient from .Models.CloudClusterStatus import CloudClusterStatus from .Models.CloudPrintJobUploadRequest import CloudPrintJobUploadRequest @@ -43,9 +44,6 @@ class T: COULD_NOT_EXPORT = _I18N_CATALOG.i18nc("@info:status", "Could not export print job.") - SENDING_DATA_TEXT = _I18N_CATALOG.i18nc("@info:status", "Sending data to remote cluster") - SENDING_DATA_TITLE = _I18N_CATALOG.i18nc("@info:status", "Sending data to remote cluster") - ERROR = _I18N_CATALOG.i18nc("@info:title", "Error") UPLOAD_ERROR = _I18N_CATALOG.i18nc("@info:text", "Could not upload the data to the printer.") @@ -68,7 +66,7 @@ class T: class CloudOutputDevice(NetworkedPrinterOutputDevice): # The interval with which the remote clusters are checked - CHECK_CLUSTER_INTERVAL = 4.0 # seconds + CHECK_CLUSTER_INTERVAL = 5.0 # seconds # Signal triggered when the print jobs in the queue were changed. printJobsChanged = pyqtSignal() @@ -109,9 +107,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): self._number_of_extruders = 2 # All networked printers are dual-extrusion Ultimaker machines. # We only allow a single upload at a time. - self._sending_job = False - # TODO: handle progress messages in another class. - self._progress_message = None # type: Optional[Message] + self._progress = CloudProgressMessage() # Keep server string of the last generated time to avoid updating models more than once for the same response self._received_printers = None # type: Optional[List[CloudClusterPrinterStatus]] @@ -149,7 +145,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): file_handler: Optional[FileHandler] = None, **kwargs: str) -> None: # Show an error message if we're already sending a job. - if self._sending_job: + if self._progress.visible: self._onUploadError(T.BLOCKED_UPLOADING) return @@ -286,53 +282,31 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # \param mesh: The bytes to upload. # \param job_response: The response received from the cloud API. def _onPrintJobCreated(self, mesh: bytes, job_response: CloudPrintJobResponse) -> None: - self._api.uploadMesh(job_response, mesh, self._onPrintJobUploaded, self._updateUploadProgress, - lambda _: self._onUploadError(T.UPLOAD_ERROR)) + self._progress.show() + self._api.uploadMesh(job_response, mesh, lambda: self._onPrintJobUploaded(job_response.job_id), + self._progress.update, self._onUploadError) ## Requests the print to be sent to the printer when we finished uploading the mesh. # \param job_id: The ID of the job. def _onPrintJobUploaded(self, job_id: str) -> None: self._api.requestPrint(self._device_id, job_id, self._onUploadSuccess) - ## Updates the progress of the mesh upload. - # \param progress: The amount of percentage points uploaded until now (0-100). - def _updateUploadProgress(self, progress: int) -> None: - if not self._progress_message: - self._progress_message = Message( - text = T.SENDING_DATA_TEXT, - title = T.SENDING_DATA_TITLE, - progress = -1, - lifetime = 0, - dismissable = False, - use_inactivity_timer = False - ) - self._progress_message.setProgress(progress) - self._progress_message.show() - - ## Hides the upload progress bar - def _resetUploadProgress(self) -> None: - if self._progress_message: - self._progress_message.hide() - self._progress_message = None - ## Displays the given message if uploading the mesh has failed # \param message: The message to display. - def _onUploadError(self, message: str = None) -> None: - self._resetUploadProgress() - if message: - Message( - text = message, - title = T.ERROR, - lifetime = 10 - ).show() - self._sending_job = False # the upload has finished so we're not sending a job anymore + def _onUploadError(self, message = None) -> None: + self._progress.hide() + Message( + text = message or T.UPLOAD_ERROR, + title = T.ERROR, + lifetime = 10 + ).show() self.writeError.emit() ## Shows a message when the upload has succeeded # \param response: The response from the cloud API. def _onUploadSuccess(self, response: CloudPrintResponse) -> None: Logger.log("i", "The cluster will be printing this print job with the ID %s", response.cluster_job_id) - self._resetUploadProgress() + self._progress.hide() Message( text = T.UPLOAD_SUCCESS_TEXT, title = T.UPLOAD_SUCCESS_TITLE, diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py new file mode 100644 index 0000000000..e3e0cefc0c --- /dev/null +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py @@ -0,0 +1,37 @@ +# Copyright (c) 2018 Ultimaker B.V. +# Cura is released under the terms of the LGPLv3 or higher. +from UM import i18nCatalog +from UM.Message import Message + + +## Class that contains all the translations for this module. +class T: + _I18N_CATALOG = i18nCatalog("cura") + + SENDING_DATA_TEXT = _I18N_CATALOG.i18nc("@info:status", "Sending data to remote cluster") + SENDING_DATA_TITLE = _I18N_CATALOG.i18nc("@info:status", "Sending data to remote cluster") + + +class CloudProgressMessage(Message): + def __init__(self): + super().__init__( + text = T.SENDING_DATA_TEXT, + title = T.SENDING_DATA_TITLE, + progress = -1, + lifetime = 0, + dismissable = False, + use_inactivity_timer = False + ) + + def show(self): + self.setProgress(0) + super().show() + + def update(self, percentage: int) -> None: + if not self._visible: + super().show() + self.setProgress(percentage) + + @property + def visible(self) -> bool: + return self._visible diff --git a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py b/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py new file mode 100644 index 0000000000..52b8e5c2d7 --- /dev/null +++ b/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py @@ -0,0 +1,94 @@ +# Copyright (c) 2018 Ultimaker B.V. +# !/usr/bin/env python +# -*- coding: utf-8 -*- +from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply +from typing import Optional, Callable, Any, Tuple + +from UM.Logger import Logger +from cura.NetworkClient import NetworkClient + + +class ResumableUpload(NetworkClient): + MAX_RETRIES = 10 + BYTES_PER_REQUEST = 256 * 1024 + RETRY_HTTP_CODES = {500, 502, 503, 504} + + ## Creates a resumable upload + # \param url: The URL to which we shall upload. + # \param content_length: The total content length of the file, in bytes. + # \param http_method: The HTTP method to be used, e.g. "POST" or "PUT". + # \param timeout: The timeout for each chunk upload. Important: If None, no timeout is applied at all. + def __init__(self, url: str, content_type: str, data: bytes, + on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): + super().__init__() + self._url = url + self._content_type = content_type + self._data = data + + self._on_finished = on_finished + self._on_progress = on_progress + self._on_error = on_error + + self._sent_bytes = 0 + self._retries = 0 + self._finished = False + + ## We override _createEmptyRequest in order to add the user credentials. + # \param url: The URL to request + # \param content_type: The type of the body contents. + def _createEmptyRequest(self, path: str, content_type: Optional[str] = "application/json") -> QNetworkRequest: + request = super()._createEmptyRequest(path, content_type = self._content_type) + + first_byte, last_byte = self._chunkRange() + content_range = "bytes {}-{}/{}".format(first_byte, last_byte - 1, len(self._data)) + request.setRawHeader(b"Content-Range", content_range.encode()) + Logger.log("i", "Uploading %s to %s", content_range, self._url) + + return request + + def _chunkRange(self) -> Tuple[int, int]: + last_byte = min(len(self._data), self._sent_bytes + self.BYTES_PER_REQUEST) + return self._sent_bytes, last_byte + + def start(self) -> None: + self._uploadChunk() + + def _uploadChunk(self) -> None: + if self._finished: + raise ValueError("The upload is already finished") + + first_byte, last_byte = self._chunkRange() + Logger.log("i", "PUT %s - %s", first_byte, last_byte) + self.put(self._url, data = self._data[first_byte:last_byte], content_type = self._content_type, + on_finished = self.finishedCallback, on_progress = self.progressCallback) + + def progressCallback(self, bytes_sent: int, bytes_total: int) -> None: + if bytes_total: + self._on_progress(int((self._sent_bytes + bytes_sent) / len(self._data) * 100)) + + def finishedCallback(self, reply: QNetworkReply) -> None: + status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) + + if self._retries < self.MAX_RETRIES and status_code in self.RETRY_HTTP_CODES: + self._retries += 1 + Logger.log("i", "Retrying %s/%s request %s", tries, self.MAX_RETRIES, request.url) + self._uploadChunk() + return + + body = bytes(reply.readAll()).decode() + Logger.log("w", "status_code: %s, Headers: %s, body: %s", status_code, + [bytes(header).decode() for header in reply.rawHeaderList()], body) + + if status_code > 308: + self._finished = True + Logger.log("e", "Received error while uploading: %s", body) + self._on_error() + return + + first_byte, last_byte = self._chunkRange() + self._sent_bytes += last_byte - first_byte + self._finished = self._sent_bytes >= len(self._data) + if self._finished: + self._on_finished() + else: + self._uploadChunk() diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py b/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py index 94cc239c0a..60627cbe7c 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py @@ -76,7 +76,8 @@ class NetworkManagerMock: ## Emits the signal that the reply is ready to all prepared replies. def flushReplies(self) -> None: - for reply in self.replies.values(): + for key, reply in self.replies.items(): + Logger.log("i", "Flushing reply to {} {}", *key) self.finished.emit(reply) self.reset() diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py index d673554640..e377627465 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py @@ -89,16 +89,17 @@ class TestCloudApiClient(TestCase): data = parseFixture("putJobUploadResponse")["data"] upload_response = CloudPrintJobResponse(**data) - self.network.prepareReply("PUT", upload_response.upload_url, 200, - b'{ data : "" }') # Network client doesn't look into the reply + # Network client doesn't look into the reply + self.network.prepareReply("PUT", upload_response.upload_url, 200, b'{}') - self.api.uploadMesh(upload_response, b'', lambda job_id: results.append(job_id), - progress.advance, progress.error) + mesh = ("1234" * 100000).encode() + self.api.uploadMesh(upload_response, mesh, lambda: results.append("sent"), progress.advance, progress.error) - self.network.flushReplies() + for _ in range(10): + self.network.flushReplies() + self.network.prepareReply("PUT", upload_response.upload_url, 200, b'{}') - self.assertEqual(len(results), 1) - self.assertEqual(results[0], upload_response.job_id) + self.assertEqual(["sent"], results) def test_requestPrint(self, network_mock): network_mock.return_value = self.network From 4dc8edb99625de2c4b1efbe96eb3170e034b795b Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 14 Dec 2018 12:48:40 +0100 Subject: [PATCH 02/16] STAR-322: Fixing the multipart upload --- cura/NetworkClient.py | 82 +++++++------------ cura/OAuth2/AuthorizationService.py | 2 + .../src/Cloud/CloudOutputDevice.py | 12 +-- .../src/Cloud/CloudOutputDeviceManager.py | 34 ++++++-- .../src/Cloud/ResumableUpload.py | 53 ++++++++---- .../src/UM3OutputDevicePlugin.py | 2 + 6 files changed, 105 insertions(+), 80 deletions(-) diff --git a/cura/NetworkClient.py b/cura/NetworkClient.py index 6abf4f3d47..b455d03db0 100644 --- a/cura/NetworkClient.py +++ b/cura/NetworkClient.py @@ -1,7 +1,7 @@ # Copyright (c) 2018 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. from time import time -from typing import Optional, Dict, Callable, List, Union +from typing import Optional, Dict, Callable, List, Union, Tuple from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QHttpMultiPart, QNetworkRequest, QHttpPart, \ @@ -16,50 +16,63 @@ from UM.Logger import Logger class NetworkClient: def __init__(self) -> None: - + # Network manager instance to use for this client. self._manager = None # type: Optional[QNetworkAccessManager] - + # Timings. self._last_manager_create_time = None # type: Optional[float] self._last_response_time = None # type: Optional[float] self._last_request_time = None # type: Optional[float] - + # The user agent of Cura. application = Application.getInstance() self._user_agent = "%s/%s " % (application.getApplicationName(), application.getVersion()) # Uses to store callback methods for finished network requests. # This allows us to register network calls with a callback directly instead of having to dissect the reply. - self._on_finished_callbacks = {} # type: Dict[str, Callable[[QNetworkReply], None]] + # The key is created out of a tuple (operation, url) + self._on_finished_callbacks = {} # type: Dict[Tuple[int, str], Callable[[QNetworkReply], None]] # QHttpMultiPart objects need to be kept alive and not garbage collected during the # HTTP which uses them. We hold references to these QHttpMultiPart objects here. self._kept_alive_multiparts = {} # type: Dict[QNetworkReply, QHttpMultiPart] - ## Creates a network manager with all the required properties and event bindings. - def _createNetworkManager(self) -> None: + ## Creates a network manager if needed, with all the required properties and event bindings. + def start(self) -> None: if self._manager: - self._manager.finished.disconnect(self.__handleOnFinished) - self._manager.authenticationRequired.disconnect(self._onAuthenticationRequired) + return self._manager = QNetworkAccessManager() - self._manager.finished.connect(self.__handleOnFinished) + self._manager.finished.connect(self._handleOnFinished) self._last_manager_create_time = time() self._manager.authenticationRequired.connect(self._onAuthenticationRequired) + ## Destroys the network manager and event bindings. + def stop(self) -> None: + if not self._manager: + return + self._manager.finished.disconnect(self._handleOnFinished) + self._manager.authenticationRequired.disconnect(self._onAuthenticationRequired) + self._manager = None + ## Create a new empty network request. # Automatically adds the required HTTP headers. # \param url: The URL to request # \param content_type: The type of the body contents. def _createEmptyRequest(self, url: str, content_type: Optional[str] = "application/json") -> QNetworkRequest: + if not self._manager: + self.start() # make sure the manager is created request = QNetworkRequest(QUrl(url)) if content_type: request.setHeader(QNetworkRequest.ContentTypeHeader, content_type) request.setHeader(QNetworkRequest.UserAgentHeader, self._user_agent) + self._last_request_time = time() return request ## Executes the correct callback method when a network request finishes. - def __handleOnFinished(self, reply: QNetworkReply) -> None: + def _handleOnFinished(self, reply: QNetworkReply) -> None: + + Logger.log("i", "On finished %s %s", reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url().toString()) # Due to garbage collection, we need to cache certain bits of post operations. # As we don't want to keep them around forever, delete them if we get a reply. @@ -76,7 +89,7 @@ class NetworkClient: # Find the right callback and execute it. # It always takes the full reply as single parameter. - callback_key = reply.url().toString() + str(reply.operation()) + callback_key = reply.operation(), reply.url().toString() if callback_key in self._on_finished_callbacks: self._on_finished_callbacks[callback_key](reply) else: @@ -87,12 +100,6 @@ class NetworkClient: if reply in self._kept_alive_multiparts: del self._kept_alive_multiparts[reply] - ## Makes sure the network manager is created. - def _validateManager(self) -> None: - if self._manager is None: - self._createNetworkManager() - assert self._manager is not None - ## Callback for when the network manager detects that authentication is required but was not given. @staticmethod def _onAuthenticationRequired(reply: QNetworkReply, authenticator: QAuthenticator) -> None: @@ -102,7 +109,7 @@ class NetworkClient: def _registerOnFinishedCallback(self, reply: QNetworkReply, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: if on_finished is not None: - self._on_finished_callbacks[reply.url().toString() + str(reply.operation())] = on_finished + self._on_finished_callbacks[reply.operation(), reply.url().toString()] = on_finished ## Add a part to a Multi-Part form. @staticmethod @@ -133,33 +140,23 @@ class NetworkClient: def put(self, url: str, data: Union[str, bytes], content_type: Optional[str] = None, on_finished: Optional[Callable[[QNetworkReply], None]] = None, on_progress: Optional[Callable[[int, int], None]] = None) -> None: - self._validateManager() - request = self._createEmptyRequest(url, content_type = content_type) - self._last_request_time = time() - - if not self._manager: - return Logger.log("e", "No network manager was created to execute the PUT call with.") body = data if isinstance(data, bytes) else data.encode() # type: bytes reply = self._manager.put(request, body) self._registerOnFinishedCallback(reply, on_finished) if on_progress is not None: + # TODO: Do we need to disconnect() as well? reply.uploadProgress.connect(on_progress) + reply.finished.connect(lambda r: Logger.log("i", "On finished %s %s", url, r)) + reply.error.connect(lambda r: Logger.log("i", "On error %s %s", url, r)) ## Sends a delete request to the given path. # url: The path after the API prefix. # on_finished: The function to be call when the response is received. def delete(self, url: str, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: - self._validateManager() - request = self._createEmptyRequest(url) - self._last_request_time = time() - - if not self._manager: - return Logger.log("e", "No network manager was created to execute the DELETE call with.") - reply = self._manager.deleteResource(request) self._registerOnFinishedCallback(reply, on_finished) @@ -167,14 +164,7 @@ class NetworkClient: # \param url: The path after the API prefix. # \param on_finished: The function to be call when the response is received. def get(self, url: str, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: - self._validateManager() - request = self._createEmptyRequest(url) - self._last_request_time = time() - - if not self._manager: - return Logger.log("e", "No network manager was created to execute the GET call with.") - reply = self._manager.get(request) self._registerOnFinishedCallback(reply, on_finished) @@ -186,13 +176,7 @@ class NetworkClient: def post(self, url: str, data: Union[str, bytes], on_finished: Optional[Callable[[QNetworkReply], None]], on_progress: Optional[Callable[[int, int], None]] = None) -> None: - self._validateManager() - request = self._createEmptyRequest(url) - self._last_request_time = time() - - if not self._manager: - return Logger.log("e", "Could not find manager.") body = data if isinstance(data, bytes) else data.encode() # type: bytes reply = self._manager.post(request, body) @@ -213,20 +197,12 @@ class NetworkClient: def postFormWithParts(self, target: str, parts: List[QHttpPart], on_finished: Optional[Callable[[QNetworkReply], None]], on_progress: Optional[Callable[[int, int], None]] = None) -> Optional[QNetworkReply]: - self._validateManager() - request = self._createEmptyRequest(target, content_type = None) multi_post_part = QHttpMultiPart(QHttpMultiPart.FormDataType) for part in parts: multi_post_part.append(part) - self._last_request_time = time() - - if not self._manager: - Logger.log("e", "No network manager was created to execute the POST call with.") - return None - reply = self._manager.post(request, multi_post_part) self._kept_alive_multiparts[reply] = multi_post_part diff --git a/cura/OAuth2/AuthorizationService.py b/cura/OAuth2/AuthorizationService.py index 4355891139..21dbbe8248 100644 --- a/cura/OAuth2/AuthorizationService.py +++ b/cura/OAuth2/AuthorizationService.py @@ -54,6 +54,8 @@ class AuthorizationService: self._user_profile = self._parseJWT() if not self._user_profile: # If there is still no user profile from the JWT, we have to log in again. + Logger.log("w", "The user profile could not be loaded. The user must log in again!") + self.deleteAuthData() return None return self._user_profile diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index e75989a6a8..83b5bed16b 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -66,7 +66,7 @@ class T: class CloudOutputDevice(NetworkedPrinterOutputDevice): # The interval with which the remote clusters are checked - CHECK_CLUSTER_INTERVAL = 5.0 # seconds + CHECK_CLUSTER_INTERVAL = 50.0 # seconds # Signal triggered when the print jobs in the queue were changed. printJobsChanged = pyqtSignal() @@ -150,7 +150,6 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): return # Indicate we have started sending a job. - self._sending_job = True self.writeStarted.emit(self) mesh_format = MeshFormatHandler(file_handler, self.firmwareVersion) @@ -173,6 +172,8 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): if self._last_response_time and time() - self._last_response_time < self.CHECK_CLUSTER_INTERVAL: return # avoid calling the cloud too often + Logger.log("i", "Requesting update for %s after %s", self._device_id, + self._last_response_time and time() - self._last_response_time) if self._account.isLoggedIn: self.setAuthenticationState(AuthState.Authenticated) self._api.getClusterStatus(self._device_id, self._onStatusCallFinished) @@ -183,6 +184,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # Contains both printers and print jobs statuses in a single response. def _onStatusCallFinished(self, status: CloudClusterStatus) -> None: # Update all data from the cluster. + self._last_response_time = time() if self._received_printers != status.printers: self._received_printers = status.printers self._updatePrinters(status.printers) @@ -289,7 +291,8 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Requests the print to be sent to the printer when we finished uploading the mesh. # \param job_id: The ID of the job. def _onPrintJobUploaded(self, job_id: str) -> None: - self._api.requestPrint(self._device_id, job_id, self._onUploadSuccess) + self._progress.update(100) + self._api.requestPrint(self._device_id, job_id, self._onPrintRequested) ## Displays the given message if uploading the mesh has failed # \param message: The message to display. @@ -304,7 +307,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Shows a message when the upload has succeeded # \param response: The response from the cloud API. - def _onUploadSuccess(self, response: CloudPrintResponse) -> None: + def _onPrintRequested(self, response: CloudPrintResponse) -> None: Logger.log("i", "The cluster will be printing this print job with the ID %s", response.cluster_job_id) self._progress.hide() Message( @@ -312,7 +315,6 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): title = T.UPLOAD_SUCCESS_TITLE, lifetime = 5 ).show() - self._sending_job = False # the upload has finished so we're not sending a job anymore self.writeFinished.emit() ## Gets the remote printers. diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py index c9b30d7c79..68b5f99bba 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py @@ -26,7 +26,7 @@ class CloudOutputDeviceManager: META_CLUSTER_ID = "um_cloud_cluster_id" # The interval with which the remote clusters are checked - CHECK_CLUSTER_INTERVAL = 5.0 # seconds + CHECK_CLUSTER_INTERVAL = 50.0 # seconds # The translation catalog for this device. I18N_CATALOG = i18nCatalog("cura") @@ -39,26 +39,22 @@ class CloudOutputDeviceManager: self._output_device_manager = application.getOutputDeviceManager() self._account = application.getCuraAPI().account # type: Account - self._account.loginStateChanged.connect(self._onLoginStateChanged) self._api = CloudApiClient(self._account, self._onApiError) - # When switching machines we check if we have to activate a remote cluster. - application.globalContainerStackChanged.connect(self._connectToActiveMachine) - # create a timer to update the remote cluster list self._update_timer = QTimer(application) self._update_timer.setInterval(int(self.CHECK_CLUSTER_INTERVAL * 1000)) self._update_timer.setSingleShot(False) - self._update_timer.timeout.connect(self._getRemoteClusters) - # Make sure the timer is started in case we missed the loginChanged signal - self._onLoginStateChanged(self._account.isLoggedIn) + self._running = False # Called when the uses logs in or out def _onLoginStateChanged(self, is_logged_in: bool) -> None: + Logger.log("i", "Log in state changed to %s", is_logged_in) if is_logged_in: if not self._update_timer.isActive(): self._update_timer.start() + self._getRemoteClusters() else: if self._update_timer.isActive(): self._update_timer.stop() @@ -136,3 +132,25 @@ class CloudOutputDeviceManager: lifetime = 10, dismissable = True ).show() + + def start(self): + if self._running: + return + application = CuraApplication.getInstance() + self._account.loginStateChanged.connect(self._onLoginStateChanged) + # When switching machines we check if we have to activate a remote cluster. + application.globalContainerStackChanged.connect(self._connectToActiveMachine) + self._update_timer.timeout.connect(self._getRemoteClusters) + self._api.start() + self._onLoginStateChanged(is_logged_in = self._account.isLoggedIn) + + def stop(self): + if not self._running: + return + application = CuraApplication.getInstance() + self._account.loginStateChanged.disconnect(self._onLoginStateChanged) + # When switching machines we check if we have to activate a remote cluster. + application.globalContainerStackChanged.disconnect(self._connectToActiveMachine) + self._update_timer.timeout.disconnect(self._getRemoteClusters) + self._api.stop() + self._onLoginStateChanged(is_logged_in = False) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py b/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py index 52b8e5c2d7..e2052c33c8 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py @@ -51,44 +51,69 @@ class ResumableUpload(NetworkClient): return self._sent_bytes, last_byte def start(self) -> None: + super().start() + if self._finished: + self._sent_bytes = 0 + self._retries = 0 + self._finished = False self._uploadChunk() + def stop(self): + super().stop() + Logger.log("i", "Stopped uploading") + self._finished = True + def _uploadChunk(self) -> None: if self._finished: raise ValueError("The upload is already finished") first_byte, last_byte = self._chunkRange() - Logger.log("i", "PUT %s - %s", first_byte, last_byte) - self.put(self._url, data = self._data[first_byte:last_byte], content_type = self._content_type, - on_finished = self.finishedCallback, on_progress = self.progressCallback) + # self.put(self._url, data = self._data[first_byte:last_byte], content_type = self._content_type, + # on_finished = self.finishedCallback, on_progress = self._progressCallback) + request = self._createEmptyRequest(self._url, content_type=self._content_type) - def progressCallback(self, bytes_sent: int, bytes_total: int) -> None: + reply = self._manager.put(request, self._data[first_byte:last_byte]) + reply.finished.connect(lambda: self._finishedCallback(reply)) + reply.uploadProgress.connect(self._progressCallback) + reply.error.connect(self._errorCallback) + if reply.isFinished(): + self._finishedCallback(reply) + + def _progressCallback(self, bytes_sent: int, bytes_total: int) -> None: + Logger.log("i", "Progress callback %s / %s", bytes_sent, bytes_total) if bytes_total: self._on_progress(int((self._sent_bytes + bytes_sent) / len(self._data) * 100)) - def finishedCallback(self, reply: QNetworkReply) -> None: + def _errorCallback(self, reply: QNetworkReply) -> None: + body = bytes(reply.readAll()).decode() + Logger.log("e", "Received error while uploading: %s", body) + self.stop() + self._on_error() + + def _finishedCallback(self, reply: QNetworkReply) -> None: + Logger.log("i", "Finished callback %s %s", + reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url().toString()) + status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if self._retries < self.MAX_RETRIES and status_code in self.RETRY_HTTP_CODES: self._retries += 1 - Logger.log("i", "Retrying %s/%s request %s", tries, self.MAX_RETRIES, request.url) + Logger.log("i", "Retrying %s/%s request %s", self._retries, self.MAX_RETRIES, reply.url().toString()) self._uploadChunk() return + if status_code > 308: + self._errorCallback(reply) + return + body = bytes(reply.readAll()).decode() Logger.log("w", "status_code: %s, Headers: %s, body: %s", status_code, [bytes(header).decode() for header in reply.rawHeaderList()], body) - if status_code > 308: - self._finished = True - Logger.log("e", "Received error while uploading: %s", body) - self._on_error() - return - first_byte, last_byte = self._chunkRange() self._sent_bytes += last_byte - first_byte - self._finished = self._sent_bytes >= len(self._data) - if self._finished: + if self._sent_bytes >= len(self._data): + self.stop() self._on_finished() else: self._uploadChunk() diff --git a/plugins/UM3NetworkPrinting/src/UM3OutputDevicePlugin.py b/plugins/UM3NetworkPrinting/src/UM3OutputDevicePlugin.py index 6a80ae046e..52fbf31e3c 100644 --- a/plugins/UM3NetworkPrinting/src/UM3OutputDevicePlugin.py +++ b/plugins/UM3NetworkPrinting/src/UM3OutputDevicePlugin.py @@ -86,6 +86,7 @@ class UM3OutputDevicePlugin(OutputDevicePlugin): ## Start looking for devices on network. def start(self): self.startDiscovery() + self._cloud_output_device_manager.start() def startDiscovery(self): self.stop() @@ -142,6 +143,7 @@ class UM3OutputDevicePlugin(OutputDevicePlugin): if self._zero_conf is not None: Logger.log("d", "zeroconf close...") self._zero_conf.close() + self._cloud_output_device_manager.stop() def removeManualDevice(self, key, address = None): if key in self._discovered_devices: From 2f08854097dcf3895e2d320752c8afa6a67296f1 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 14 Dec 2018 14:50:15 +0100 Subject: [PATCH 03/16] STAR-322: Using QNetworkReply.finished signal instead of QNetworkAccessManager.finished --- cura/NetworkClient.py | 73 ++++++------------- .../src/Cloud/CloudApiClient.py | 32 +++++--- .../src/Cloud/CloudOutputDevice.py | 2 - .../src/Cloud/CloudOutputDeviceManager.py | 11 ++- .../src/Cloud/ResumableUpload.py | 54 +++++++------- .../tests/Cloud/NetworkManagerMock.py | 23 +++++- .../tests/Cloud/TestCloudApiClient.py | 55 ++++++-------- .../tests/Cloud/TestCloudOutputDevice.py | 21 +++--- .../Cloud/TestCloudOutputDeviceManager.py | 45 ++++++------ 9 files changed, 152 insertions(+), 164 deletions(-) diff --git a/cura/NetworkClient.py b/cura/NetworkClient.py index b455d03db0..4c43e58c4f 100644 --- a/cura/NetworkClient.py +++ b/cura/NetworkClient.py @@ -1,9 +1,9 @@ # Copyright (c) 2018 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. from time import time -from typing import Optional, Dict, Callable, List, Union, Tuple +from typing import Optional, Dict, Callable, List, Union -from PyQt5.QtCore import QUrl +from PyQt5.QtCore import QUrl, QObject from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QHttpMultiPart, QNetworkRequest, QHttpPart, \ QAuthenticator @@ -13,9 +13,10 @@ from UM.Logger import Logger ## Abstraction of QNetworkAccessManager for easier networking in Cura. # This was originally part of NetworkedPrinterOutputDevice but was moved out for re-use in other classes. -class NetworkClient: +class NetworkClient(QObject): def __init__(self) -> None: + super().__init__() # Network manager instance to use for this client. self._manager = None # type: Optional[QNetworkAccessManager] @@ -29,11 +30,6 @@ class NetworkClient: application = Application.getInstance() self._user_agent = "%s/%s " % (application.getApplicationName(), application.getVersion()) - # Uses to store callback methods for finished network requests. - # This allows us to register network calls with a callback directly instead of having to dissect the reply. - # The key is created out of a tuple (operation, url) - self._on_finished_callbacks = {} # type: Dict[Tuple[int, str], Callable[[QNetworkReply], None]] - # QHttpMultiPart objects need to be kept alive and not garbage collected during the # HTTP which uses them. We hold references to these QHttpMultiPart objects here. self._kept_alive_multiparts = {} # type: Dict[QNetworkReply, QHttpMultiPart] @@ -43,7 +39,6 @@ class NetworkClient: if self._manager: return self._manager = QNetworkAccessManager() - self._manager.finished.connect(self._handleOnFinished) self._last_manager_create_time = time() self._manager.authenticationRequired.connect(self._onAuthenticationRequired) @@ -51,7 +46,6 @@ class NetworkClient: def stop(self) -> None: if not self._manager: return - self._manager.finished.disconnect(self._handleOnFinished) self._manager.authenticationRequired.disconnect(self._onAuthenticationRequired) self._manager = None @@ -69,32 +63,6 @@ class NetworkClient: self._last_request_time = time() return request - ## Executes the correct callback method when a network request finishes. - def _handleOnFinished(self, reply: QNetworkReply) -> None: - - Logger.log("i", "On finished %s %s", reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url().toString()) - - # Due to garbage collection, we need to cache certain bits of post operations. - # As we don't want to keep them around forever, delete them if we get a reply. - if reply.operation() == QNetworkAccessManager.PostOperation: - self._clearCachedMultiPart(reply) - - # No status code means it never even reached remote. - if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) is None: - return - - # Not used by this class itself, but children might need it for better network handling. - # An example of this is the _update method in the NetworkedPrinterOutputDevice. - self._last_response_time = time() - - # Find the right callback and execute it. - # It always takes the full reply as single parameter. - callback_key = reply.operation(), reply.url().toString() - if callback_key in self._on_finished_callbacks: - self._on_finished_callbacks[callback_key](reply) - else: - Logger.log("w", "Received reply to URL %s but no callbacks are registered", reply.url()) - ## Removes all cached Multi-Part items. def _clearCachedMultiPart(self, reply: QNetworkReply) -> None: if reply in self._kept_alive_multiparts: @@ -105,12 +73,6 @@ class NetworkClient: def _onAuthenticationRequired(reply: QNetworkReply, authenticator: QAuthenticator) -> None: Logger.log("w", "Request to {} required authentication but was not given".format(reply.url().toString())) - ## Register a method to be executed when the associated network request finishes. - def _registerOnFinishedCallback(self, reply: QNetworkReply, - on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: - if on_finished is not None: - self._on_finished_callbacks[reply.operation(), reply.url().toString()] = on_finished - ## Add a part to a Multi-Part form. @staticmethod def _createFormPart(content_header: str, data: bytes, content_type: Optional[str] = None) -> QHttpPart: @@ -144,13 +106,10 @@ class NetworkClient: body = data if isinstance(data, bytes) else data.encode() # type: bytes reply = self._manager.put(request, body) - self._registerOnFinishedCallback(reply, on_finished) - + callback = self._createCallback(reply, on_finished) + reply.finished.connect(callback) if on_progress is not None: - # TODO: Do we need to disconnect() as well? reply.uploadProgress.connect(on_progress) - reply.finished.connect(lambda r: Logger.log("i", "On finished %s %s", url, r)) - reply.error.connect(lambda r: Logger.log("i", "On error %s %s", url, r)) ## Sends a delete request to the given path. # url: The path after the API prefix. @@ -158,7 +117,8 @@ class NetworkClient: def delete(self, url: str, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: request = self._createEmptyRequest(url) reply = self._manager.deleteResource(request) - self._registerOnFinishedCallback(reply, on_finished) + callback = self._createCallback(reply, on_finished) + reply.finished.connect(callback) ## Sends a get request to the given path. # \param url: The path after the API prefix. @@ -166,7 +126,8 @@ class NetworkClient: def get(self, url: str, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: request = self._createEmptyRequest(url) reply = self._manager.get(request) - self._registerOnFinishedCallback(reply, on_finished) + callback = self._createCallback(reply, on_finished) + reply.finished.connect(callback) ## Sends a post request to the given path. # \param url: The path after the API prefix. @@ -180,9 +141,10 @@ class NetworkClient: body = data if isinstance(data, bytes) else data.encode() # type: bytes reply = self._manager.post(request, body) + callback = self._createCallback(reply, on_finished) + reply.finished.connect(callback) if on_progress is not None: reply.uploadProgress.connect(on_progress) - self._registerOnFinishedCallback(reply, on_finished) ## Does a POST request with form data to the given URL. def postForm(self, url: str, header_data: str, body_data: bytes, @@ -205,10 +167,19 @@ class NetworkClient: reply = self._manager.post(request, multi_post_part) + def callback(): + on_finished(reply) + self._clearCachedMultiPart(reply) + + reply.finished.connect(callback) + self._kept_alive_multiparts[reply] = multi_post_part if on_progress is not None: reply.uploadProgress.connect(on_progress) - self._registerOnFinishedCallback(reply, on_finished) return reply + + @staticmethod + def _createCallback(reply: QNetworkReply, on_finished: Optional[Callable[[QNetworkReply], None]] = None): + return lambda: on_finished(reply) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index 2637f17010..7c3c08e044 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -4,11 +4,11 @@ import json from json import JSONDecodeError from typing import Callable, List, Type, TypeVar, Union, Optional, Tuple, Dict, Any -from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply +from PyQt5.QtCore import QObject, QUrl +from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManager from UM.Logger import Logger from cura.API import Account -from cura.NetworkClient import NetworkClient from .ResumableUpload import ResumableUpload from ..Models import BaseModel from .Models.CloudClusterResponse import CloudClusterResponse @@ -21,7 +21,7 @@ from .Models.CloudPrintJobResponse import CloudPrintJobResponse ## The cloud API client is responsible for handling the requests and responses from the cloud. # Each method should only handle models instead of exposing Any HTTP details. -class CloudApiClient(NetworkClient): +class CloudApiClient: # The cloud URL to use for this remote cluster. # TODO: Make sure that this URL goes to the live api before release @@ -34,6 +34,7 @@ class CloudApiClient(NetworkClient): # \param on_error: The callback to be called whenever we receive errors from the server. def __init__(self, account: Account, on_error: Callable[[List[CloudErrorObject]], None]) -> None: super().__init__() + self._manager = QNetworkAccessManager() self._account = account self._on_error = on_error @@ -46,14 +47,18 @@ class CloudApiClient(NetworkClient): # \param on_finished: The function to be called after the result is parsed. def getClusters(self, on_finished: Callable[[List[CloudClusterResponse]], Any]) -> None: url = "{}/clusters".format(self.CLUSTER_API_ROOT) - self.get(url, on_finished=self._wrapCallback(on_finished, CloudClusterResponse)) + reply = self._manager.get(self._createEmptyRequest(url)) + callback = self._wrapCallback(reply, on_finished, CloudClusterResponse) + reply.finished.connect(callback) ## Retrieves the status of the given cluster. # \param cluster_id: The ID of the cluster. # \param on_finished: The function to be called after the result is parsed. def getClusterStatus(self, cluster_id: str, on_finished: Callable[[CloudClusterStatus], Any]) -> None: url = "{}/clusters/{}/status".format(self.CLUSTER_API_ROOT, cluster_id) - self.get(url, on_finished=self._wrapCallback(on_finished, CloudClusterStatus)) + reply = self._manager.get(self._createEmptyRequest(url)) + callback = self._wrapCallback(reply, on_finished, CloudClusterStatus) + reply.finished.connect(callback) ## Requests the cloud to register the upload of a print job mesh. # \param request: The request object. @@ -62,7 +67,9 @@ class CloudApiClient(NetworkClient): ) -> None: url = "{}/jobs/upload".format(self.CURA_API_ROOT) body = json.dumps({"data": request.toDict()}) - self.put(url, body, on_finished=self._wrapCallback(on_finished, CloudPrintJobResponse)) + reply = self._manager.put(self._createEmptyRequest(url), body.encode()) + callback = self._wrapCallback(reply, on_finished, CloudPrintJobResponse) + reply.finished.connect(callback) ## Requests the cloud to register the upload of a print job mesh. # \param upload_response: The object received after requesting an upload with `self.requestUpload`. @@ -72,7 +79,7 @@ class CloudApiClient(NetworkClient): # \param on_error: A function to be called if the upload fails. It receives a dict with the error. def uploadMesh(self, upload_response: CloudPrintJobResponse, mesh: bytes, on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): - ResumableUpload(upload_response.upload_url, upload_response.content_type, mesh, on_finished, + ResumableUpload(self._manager, upload_response.upload_url, upload_response.content_type, mesh, on_finished, on_progress, on_error).start() # Requests a cluster to print the given print job. @@ -81,13 +88,17 @@ class CloudApiClient(NetworkClient): # \param on_finished: The function to be called after the result is parsed. def requestPrint(self, cluster_id: str, job_id: str, on_finished: Callable[[CloudPrintResponse], Any]) -> None: url = "{}/clusters/{}/print/{}".format(self.CLUSTER_API_ROOT, cluster_id, job_id) - self.post(url, data = "", on_finished=self._wrapCallback(on_finished, CloudPrintResponse)) + reply = self._manager.post(self._createEmptyRequest(url), b"") + callback = self._wrapCallback(reply, on_finished, CloudPrintResponse) + reply.finished.connect(callback) ## We override _createEmptyRequest in order to add the user credentials. # \param url: The URL to request # \param content_type: The type of the body contents. def _createEmptyRequest(self, path: str, content_type: Optional[str] = "application/json") -> QNetworkRequest: - request = super()._createEmptyRequest(path, content_type) + request = QNetworkRequest(QUrl(path)) + if content_type: + request.setHeader(QNetworkRequest.ContentTypeHeader, content_type) if self._account.isLoggedIn: request.setRawHeader(b"Authorization", "Bearer {}".format(self._account.accessToken).encode()) Logger.log("i", "Created request for URL %s. Logged in = %s", path, self._account.isLoggedIn) @@ -132,10 +143,11 @@ class CloudApiClient(NetworkClient): # \param model: The type of the model to convert the response to. It may either be a single record or a list. # \return: A function that can be passed to the def _wrapCallback(self, + reply: QNetworkReply, on_finished: Callable[[Union[Model, List[Model]]], Any], model: Type[Model], ) -> Callable[[QNetworkReply], None]: - def parse(reply: QNetworkReply) -> None: + def parse() -> None: status_code, response = self._parseReply(reply) return self._parseModels(response, on_finished, model) return parse diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index 83b5bed16b..09677d5e48 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -172,8 +172,6 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): if self._last_response_time and time() - self._last_response_time < self.CHECK_CLUSTER_INTERVAL: return # avoid calling the cloud too often - Logger.log("i", "Requesting update for %s after %s", self._device_id, - self._last_response_time and time() - self._last_response_time) if self._account.isLoggedIn: self.setAuthenticationState(AuthState.Authenticated) self._api.getClusterStatus(self._device_id, self._onStatusCallFinished) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py index 68b5f99bba..af80907f01 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py @@ -125,13 +125,14 @@ class CloudOutputDeviceManager: ## Handles an API error received from the cloud. # \param errors: The errors received def _onApiError(self, errors: List[CloudErrorObject]) -> None: - message = ". ".join(e.title for e in errors) # TODO: translate errors - Message( - text = message, + text = ". ".join(e.title for e in errors) # TODO: translate errors + message = Message( + text = text, title = self.I18N_CATALOG.i18nc("@info:title", "Error"), lifetime = 10, dismissable = True - ).show() + ) + message.show() def start(self): if self._running: @@ -141,7 +142,6 @@ class CloudOutputDeviceManager: # When switching machines we check if we have to activate a remote cluster. application.globalContainerStackChanged.connect(self._connectToActiveMachine) self._update_timer.timeout.connect(self._getRemoteClusters) - self._api.start() self._onLoginStateChanged(is_logged_in = self._account.isLoggedIn) def stop(self): @@ -152,5 +152,4 @@ class CloudOutputDeviceManager: # When switching machines we check if we have to activate a remote cluster. application.globalContainerStackChanged.disconnect(self._connectToActiveMachine) self._update_timer.timeout.disconnect(self._getRemoteClusters) - self._api.stop() self._onLoginStateChanged(is_logged_in = False) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py b/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py index e2052c33c8..5e3bc9545e 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py @@ -1,14 +1,14 @@ # Copyright (c) 2018 Ultimaker B.V. # !/usr/bin/env python # -*- coding: utf-8 -*- -from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply +from PyQt5.QtCore import QUrl +from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManager from typing import Optional, Callable, Any, Tuple from UM.Logger import Logger -from cura.NetworkClient import NetworkClient -class ResumableUpload(NetworkClient): +class ResumableUpload: MAX_RETRIES = 10 BYTES_PER_REQUEST = 256 * 1024 RETRY_HTTP_CODES = {500, 502, 503, 504} @@ -18,9 +18,9 @@ class ResumableUpload(NetworkClient): # \param content_length: The total content length of the file, in bytes. # \param http_method: The HTTP method to be used, e.g. "POST" or "PUT". # \param timeout: The timeout for each chunk upload. Important: If None, no timeout is applied at all. - def __init__(self, url: str, content_type: str, data: bytes, + def __init__(self, manager: QNetworkAccessManager, url: str, content_type: str, data: bytes, on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): - super().__init__() + self._manager = manager self._url = url self._content_type = content_type self._data = data @@ -32,13 +32,15 @@ class ResumableUpload(NetworkClient): self._sent_bytes = 0 self._retries = 0 self._finished = False + self._reply = None # type: Optional[QNetworkReply] - ## We override _createEmptyRequest in order to add the user credentials. + ## We override _createRequest in order to add the user credentials. # \param url: The URL to request # \param content_type: The type of the body contents. - def _createEmptyRequest(self, path: str, content_type: Optional[str] = "application/json") -> QNetworkRequest: - request = super()._createEmptyRequest(path, content_type = self._content_type) - + def _createRequest(self) -> QNetworkRequest: + request = QNetworkRequest(QUrl(self._url)) + request.setHeader(QNetworkRequest.ContentTypeHeader, self._content_type) + first_byte, last_byte = self._chunkRange() content_range = "bytes {}-{}/{}".format(first_byte, last_byte - 1, len(self._data)) request.setRawHeader(b"Content-Range", content_range.encode()) @@ -51,7 +53,6 @@ class ResumableUpload(NetworkClient): return self._sent_bytes, last_byte def start(self) -> None: - super().start() if self._finished: self._sent_bytes = 0 self._retries = 0 @@ -59,7 +60,6 @@ class ResumableUpload(NetworkClient): self._uploadChunk() def stop(self): - super().stop() Logger.log("i", "Stopped uploading") self._finished = True @@ -68,47 +68,43 @@ class ResumableUpload(NetworkClient): raise ValueError("The upload is already finished") first_byte, last_byte = self._chunkRange() - # self.put(self._url, data = self._data[first_byte:last_byte], content_type = self._content_type, - # on_finished = self.finishedCallback, on_progress = self._progressCallback) - request = self._createEmptyRequest(self._url, content_type=self._content_type) + request = self._createRequest() - reply = self._manager.put(request, self._data[first_byte:last_byte]) - reply.finished.connect(lambda: self._finishedCallback(reply)) - reply.uploadProgress.connect(self._progressCallback) - reply.error.connect(self._errorCallback) - if reply.isFinished(): - self._finishedCallback(reply) + self._reply = self._manager.put(request, self._data[first_byte:last_byte]) + self._reply.finished.connect(self._finishedCallback) + self._reply.uploadProgress.connect(self._progressCallback) + self._reply.error.connect(self._errorCallback) def _progressCallback(self, bytes_sent: int, bytes_total: int) -> None: Logger.log("i", "Progress callback %s / %s", bytes_sent, bytes_total) if bytes_total: self._on_progress(int((self._sent_bytes + bytes_sent) / len(self._data) * 100)) - def _errorCallback(self, reply: QNetworkReply) -> None: - body = bytes(reply.readAll()).decode() + def _errorCallback(self) -> None: + body = bytes(self._reply.readAll()).decode() Logger.log("e", "Received error while uploading: %s", body) self.stop() self._on_error() - def _finishedCallback(self, reply: QNetworkReply) -> None: + def _finishedCallback(self) -> None: Logger.log("i", "Finished callback %s %s", - reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url().toString()) + self._reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), self._reply.url().toString()) - status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) + status_code = self._reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if self._retries < self.MAX_RETRIES and status_code in self.RETRY_HTTP_CODES: self._retries += 1 - Logger.log("i", "Retrying %s/%s request %s", self._retries, self.MAX_RETRIES, reply.url().toString()) + Logger.log("i", "Retrying %s/%s request %s", self._retries, self.MAX_RETRIES, self._reply.url().toString()) self._uploadChunk() return if status_code > 308: - self._errorCallback(reply) + self._errorCallback() return - body = bytes(reply.readAll()).decode() + body = bytes(self._reply.readAll()).decode() Logger.log("w", "status_code: %s, Headers: %s, body: %s", status_code, - [bytes(header).decode() for header in reply.rawHeaderList()], body) + [bytes(header).decode() for header in self._reply.rawHeaderList()], body) first_byte, last_byte = self._chunkRange() self._sent_bytes += last_byte - first_byte diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py b/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py index 60627cbe7c..59b79fdfa6 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py @@ -4,12 +4,27 @@ import json from typing import Dict, Tuple, Union, Optional from unittest.mock import MagicMock -from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QNetworkRequest +from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest from UM.Logger import Logger from UM.Signal import Signal +class FakeSignal: + def __init__(self): + self._callbacks = [] + + def connect(self, callback): + self._callbacks.append(callback) + + def disconnect(self, callback): + self._callbacks.remove(callback) + + def emit(self, *args, **kwargs): + for callback in self._callbacks: + callback(*args, **kwargs) + + ## This class can be used to mock the QNetworkManager class and test the code using it. # After patching the QNetworkManager class, requests are prepared before they can be executed. # Any requests not prepared beforehand will cause KeyErrors. @@ -27,7 +42,7 @@ class NetworkManagerMock: ## Initializes the network manager mock. def __init__(self) -> None: # a dict with the prepared replies, using the format {(http_method, url): reply} - self.replies = {} # type: Dict[Tuple[str, str], QNetworkReply] + self.replies = {} # type: Dict[Tuple[str, str], MagicMock] self.request_bodies = {} # type: Dict[Tuple[str, str], bytes] # signals used in the network manager. @@ -64,6 +79,8 @@ class NetworkManagerMock: reply_mock.url().toString.return_value = url reply_mock.operation.return_value = self._OPERATIONS[method] reply_mock.attribute.return_value = status_code + reply_mock.finished = FakeSignal() + reply_mock.isFinished.return_value = False reply_mock.readAll.return_value = response if isinstance(response, bytes) else json.dumps(response).encode() self.replies[method, url] = reply_mock Logger.log("i", "Prepared mock {}-response to {} {}", status_code, method, url) @@ -78,6 +95,8 @@ class NetworkManagerMock: def flushReplies(self) -> None: for key, reply in self.replies.items(): Logger.log("i", "Flushing reply to {} {}", *key) + reply.isFinished.return_value = True + reply.finished.emit() self.finished.emit(reply) self.reset() diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py index e377627465..d4044726a3 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py @@ -8,6 +8,8 @@ from unittest.mock import patch, MagicMock from cura.CuraApplication import CuraApplication from src.Cloud.CloudApiClient import CloudApiClient +from src.Cloud.Models.CloudClusterResponse import CloudClusterResponse +from src.Cloud.Models.CloudClusterStatus import CloudClusterStatus from src.Cloud.Models.CloudPrintJobResponse import CloudPrintJobResponse from src.Cloud.Models.CloudPrintJobUploadRequest import CloudPrintJobUploadRequest from src.Cloud.Models.CloudErrorObject import CloudErrorObject @@ -15,7 +17,6 @@ from tests.Cloud.Fixtures import readFixture, parseFixture from .NetworkManagerMock import NetworkManagerMock -@patch("cura.NetworkClient.QNetworkAccessManager") class TestCloudApiClient(TestCase): def _errorHandler(self, errors: List[CloudErrorObject]): raise Exception("Received unexpected error: {}".format(errors)) @@ -27,15 +28,14 @@ class TestCloudApiClient(TestCase): self.app = CuraApplication.getInstance() self.network = NetworkManagerMock() - self.api = CloudApiClient(self.account, self._errorHandler) - - def test_GetClusters(self, network_mock): - network_mock.return_value = self.network + with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): + self.api = CloudApiClient(self.account, self._errorHandler) + def test_getClusters(self): result = [] - with open("{}/Fixtures/getClusters.json".format(os.path.dirname(__file__)), "rb") as f: - response = f.read() + response = readFixture("getClusters") + data = parseFixture("getClusters")["data"] self.network.prepareReply("GET", "https://api-staging.ultimaker.com/connect/v1/clusters", 200, response) # the callback is a function that adds the result of the call to getClusters to the result list @@ -43,32 +43,26 @@ class TestCloudApiClient(TestCase): self.network.flushReplies() - self.assertEqual(2, len(result)) - - def test_getClusterStatus(self, network_mock): - network_mock.return_value = self.network + self.assertEqual([CloudClusterResponse(**data[0]), CloudClusterResponse(**data[1])], result) + def test_getClusterStatus(self): result = [] - with open("{}/Fixtures/getClusterStatusResponse.json".format(os.path.dirname(__file__)), "rb") as f: - response = f.read() + response = readFixture("getClusterStatusResponse") + data = parseFixture("getClusterStatusResponse")["data"] self.network.prepareReply("GET", "https://api-staging.ultimaker.com/connect/v1/clusters/R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC/status", 200, response ) - self.api.getClusterStatus("R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC", lambda status: result.append(status)) + self.api.getClusterStatus("R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC", lambda s: result.append(s)) self.network.flushReplies() - self.assertEqual(len(result), 1) - status = result[0] + self.assertEqual([CloudClusterStatus(**data)], result) - self.assertEqual(len(status.printers), 2) - self.assertEqual(len(status.print_jobs), 1) - - def test_requestUpload(self, network_mock): - network_mock.return_value = self.network + def test_requestUpload(self): + results = [] response = readFixture("putJobUploadResponse") @@ -78,11 +72,11 @@ class TestCloudApiClient(TestCase): self.api.requestUpload(request, lambda r: results.append(r)) self.network.flushReplies() - self.assertEqual(results[0].content_type, "text/plain") - self.assertEqual(results[0].status, "uploading") + self.assertEqual(["text/plain"], [r.content_type for r in results]) + self.assertEqual(["uploading"], [r.status for r in results]) - def test_uploadMesh(self, network_mock): - network_mock.return_value = self.network + def test_uploadMesh(self): + results = [] progress = MagicMock() @@ -101,8 +95,8 @@ class TestCloudApiClient(TestCase): self.assertEqual(["sent"], results) - def test_requestPrint(self, network_mock): - network_mock.return_value = self.network + def test_requestPrint(self): + results = [] response = readFixture("postJobPrintResponse") @@ -120,7 +114,6 @@ class TestCloudApiClient(TestCase): self.network.flushReplies() - self.assertEqual(len(results), 1) - self.assertEqual(results[0].job_id, job_id) - self.assertEqual(results[0].cluster_job_id, cluster_job_id) - self.assertEqual(results[0].status, "queued") + self.assertEqual([job_id], [r.job_id for r in results]) + self.assertEqual([cluster_job_id], [r.cluster_job_id for r in results]) + self.assertEqual(["queued"], [r.status for r in results]) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py index 287f2dda98..c391dc75dd 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py @@ -13,7 +13,6 @@ from tests.Cloud.Fixtures import readFixture, parseFixture from .NetworkManagerMock import NetworkManagerMock -@patch("cura.NetworkClient.QNetworkAccessManager") class TestCloudOutputDevice(TestCase): CLUSTER_ID = "RIZ6cZbWA_Ua7RZVJhrdVfVpf0z-MqaSHQE4v8aRTtYq" JOB_ID = "ABCDefGHIjKlMNOpQrSTUvYxWZ0-1234567890abcDE=" @@ -30,7 +29,9 @@ class TestCloudOutputDevice(TestCase): self.network = NetworkManagerMock() self.account = MagicMock(isLoggedIn=True, accessToken="TestAccessToken") self.onError = MagicMock() - self.device = CloudOutputDevice(CloudApiClient(self.account, self.onError), self.CLUSTER_ID, self.HOST_NAME) + with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): + self._api = CloudApiClient(self.account, self.onError) + self.device = CloudOutputDevice(self._api, self.CLUSTER_ID, self.HOST_NAME) self.cluster_status = parseFixture("getClusterStatusResponse") self.network.prepareReply("GET", self.STATUS_URL, 200, readFixture("getClusterStatusResponse")) @@ -38,8 +39,7 @@ class TestCloudOutputDevice(TestCase): super().tearDown() self.network.flushReplies() - def test_status(self, network_mock): - network_mock.return_value = self.network + def test_status(self): self.device._update() self.network.flushReplies() @@ -69,32 +69,34 @@ class TestCloudOutputDevice(TestCase): self.assertEqual({job["name"] for job in self.cluster_status["data"]["print_jobs"]}, {job.name for job in self.device.printJobs}) - def test_remove_print_job(self, network_mock): - network_mock.return_value = self.network + def test_remove_print_job(self): self.device._update() self.network.flushReplies() self.assertEqual(1, len(self.device.printJobs)) self.cluster_status["data"]["print_jobs"].clear() self.network.prepareReply("GET", self.STATUS_URL, 200, self.cluster_status) + + self.device._last_response_time = None self.device._update() self.network.flushReplies() self.assertEqual([], self.device.printJobs) - def test_remove_printers(self, network_mock): - network_mock.return_value = self.network + def test_remove_printers(self): self.device._update() self.network.flushReplies() self.assertEqual(2, len(self.device.printers)) self.cluster_status["data"]["printers"].clear() self.network.prepareReply("GET", self.STATUS_URL, 200, self.cluster_status) + + self.device._last_response_time = None self.device._update() self.network.flushReplies() self.assertEqual([], self.device.printers) @patch("cura.CuraApplication.CuraApplication.getGlobalContainerStack") - def test_print_to_cloud(self, global_container_stack_mock, network_mock): + def test_print_to_cloud(self, global_container_stack_mock): active_machine_mock = global_container_stack_mock.return_value active_machine_mock.getMetaDataEntry.side_effect = {"file_formats": "application/gzip"}.get @@ -104,7 +106,6 @@ class TestCloudOutputDevice(TestCase): self.network.prepareReply("PUT", request_upload_response["data"]["upload_url"], 201, b"{}") self.network.prepareReply("POST", self.PRINT_URL, 200, request_print_response) - network_mock.return_value = self.network file_handler = MagicMock() file_handler.getSupportedFileTypesWrite.return_value = [{ "extension": "gcode.gz", diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py index b6bcde6e55..80dd2c7990 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py @@ -10,7 +10,6 @@ from tests.Cloud.Fixtures import parseFixture, readFixture from .NetworkManagerMock import NetworkManagerMock -@patch("cura.NetworkClient.QNetworkAccessManager") class TestCloudOutputDeviceManager(TestCase): URL = "https://api-staging.ultimaker.com/connect/v1/clusters" @@ -18,13 +17,15 @@ class TestCloudOutputDeviceManager(TestCase): super().setUp() self.app = CuraApplication.getInstance() self.network = NetworkManagerMock() - self.manager = CloudOutputDeviceManager() + with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): + self.manager = CloudOutputDeviceManager() self.clusters_response = parseFixture("getClusters") self.network.prepareReply("GET", self.URL, 200, readFixture("getClusters")) def tearDown(self): try: self._beforeTearDown() + self.manager.stop() finally: super().tearDown() @@ -47,17 +48,17 @@ class TestCloudOutputDeviceManager(TestCase): device_manager.removeOutputDevice(device["cluster_id"]) ## Runs the initial request to retrieve the clusters. - def _loadData(self, network_mock): - network_mock.return_value = self.network - self.manager._account.loginStateChanged.emit(True) - self.manager._update_timer.timeout.emit() + def _loadData(self): + self.manager.start() + self.manager._onLoginStateChanged(is_logged_in = True) + self.network.flushReplies() - def test_device_is_created(self, network_mock): + def test_device_is_created(self): # just create the cluster, it is checked at tearDown - self._loadData(network_mock) + self._loadData() - def test_device_is_updated(self, network_mock): - self._loadData(network_mock) + def test_device_is_updated(self): + self._loadData() # update the cluster from member variable, which is checked at tearDown self.clusters_response["data"][0]["host_name"] = "New host name" @@ -65,8 +66,8 @@ class TestCloudOutputDeviceManager(TestCase): self.manager._update_timer.timeout.emit() - def test_device_is_removed(self, network_mock): - self._loadData(network_mock) + def test_device_is_removed(self): + self._loadData() # delete the cluster from member variable, which is checked at tearDown del self.clusters_response["data"][1] @@ -75,41 +76,39 @@ class TestCloudOutputDeviceManager(TestCase): self.manager._update_timer.timeout.emit() @patch("cura.CuraApplication.CuraApplication.getGlobalContainerStack") - def test_device_connects_by_cluster_id(self, global_container_stack_mock, network_mock): + def test_device_connects_by_cluster_id(self, global_container_stack_mock): active_machine_mock = global_container_stack_mock.return_value cluster1, cluster2 = self.clusters_response["data"] cluster_id = cluster1["cluster_id"] active_machine_mock.getMetaDataEntry.side_effect = {"um_cloud_cluster_id": cluster_id}.get - self._loadData(network_mock) - self.network.flushReplies() + self._loadData() self.assertTrue(self.app.getOutputDeviceManager().getOutputDevice(cluster1["cluster_id"]).isConnected()) self.assertFalse(self.app.getOutputDeviceManager().getOutputDevice(cluster2["cluster_id"]).isConnected()) self.assertEquals([], active_machine_mock.setMetaDataEntry.mock_calls) @patch("cura.CuraApplication.CuraApplication.getGlobalContainerStack") - def test_device_connects_by_network_key(self, global_container_stack_mock, network_mock): + def test_device_connects_by_network_key(self, global_container_stack_mock): active_machine_mock = global_container_stack_mock.return_value cluster1, cluster2 = self.clusters_response["data"] network_key = cluster2["host_name"] + ".ultimaker.local" active_machine_mock.getMetaDataEntry.side_effect = {"um_network_key": network_key}.get - self._loadData(network_mock) - self.network.flushReplies() + self._loadData() self.assertFalse(self.app.getOutputDeviceManager().getOutputDevice(cluster1["cluster_id"]).isConnected()) self.assertTrue(self.app.getOutputDeviceManager().getOutputDevice(cluster2["cluster_id"]).isConnected()) active_machine_mock.setMetaDataEntry.assert_called_with("um_cloud_cluster_id", cluster2["cluster_id"]) - @patch("UM.Message.Message.show") - def test_api_error(self, message_mock, network_mock): + @patch("src.Cloud.CloudOutputDeviceManager.Message") + def test_api_error(self, message_mock): self.clusters_response = { "errors": [{"id": "notFound", "title": "Not found!", "http_status": "404", "code": "notFound"}] } self.network.prepareReply("GET", self.URL, 200, self.clusters_response) - self._loadData(network_mock) - self.network.flushReplies() - message_mock.assert_called_once_with() + self._loadData() + message_mock.assert_called_once_with(text='Not found!', title='Error', lifetime=10, dismissable=True) + message_mock.return_value.show.assert_called_once_with() From e815d5da8f73a939cd4a1e028fea44c65f987cc8 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 14 Dec 2018 16:02:28 +0100 Subject: [PATCH 04/16] STAR-322: Avoiding lambdas and direct callbacks to avoid gc --- cura/NetworkClient.py | 4 +- .../src/Cloud/CloudApiClient.py | 11 +++-- .../src/Cloud/CloudOutputDevice.py | 46 +++++++++++++++---- .../{ResumableUpload.py => MeshUploader.py} | 18 +++++--- .../tests/Cloud/TestCloudOutputDevice.py | 4 ++ 5 files changed, 59 insertions(+), 24 deletions(-) rename plugins/UM3NetworkPrinting/src/Cloud/{ResumableUpload.py => MeshUploader.py} (91%) diff --git a/cura/NetworkClient.py b/cura/NetworkClient.py index 4c43e58c4f..878158542a 100644 --- a/cura/NetworkClient.py +++ b/cura/NetworkClient.py @@ -3,7 +3,7 @@ from time import time from typing import Optional, Dict, Callable, List, Union -from PyQt5.QtCore import QUrl, QObject +from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QHttpMultiPart, QNetworkRequest, QHttpPart, \ QAuthenticator @@ -13,7 +13,7 @@ from UM.Logger import Logger ## Abstraction of QNetworkAccessManager for easier networking in Cura. # This was originally part of NetworkedPrinterOutputDevice but was moved out for re-use in other classes. -class NetworkClient(QObject): +class NetworkClient: def __init__(self) -> None: super().__init__() diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index 7c3c08e044..8cdedd1229 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -4,12 +4,12 @@ import json from json import JSONDecodeError from typing import Callable, List, Type, TypeVar, Union, Optional, Tuple, Dict, Any -from PyQt5.QtCore import QObject, QUrl +from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManager from UM.Logger import Logger from cura.API import Account -from .ResumableUpload import ResumableUpload +from .MeshUploader import MeshUploader from ..Models import BaseModel from .Models.CloudClusterResponse import CloudClusterResponse from .Models.CloudErrorObject import CloudErrorObject @@ -37,6 +37,7 @@ class CloudApiClient: self._manager = QNetworkAccessManager() self._account = account self._on_error = on_error + self._upload = None # type: Optional[MeshUploader] ## Gets the account used for the API. @property @@ -77,10 +78,10 @@ class CloudApiClient: # \param on_finished: The function to be called after the result is parsed. It receives the print job ID. # \param on_progress: A function to be called during upload progress. It receives a percentage (0-100). # \param on_error: A function to be called if the upload fails. It receives a dict with the error. - def uploadMesh(self, upload_response: CloudPrintJobResponse, mesh: bytes, on_finished: Callable[[], Any], + def uploadMesh(self, print_job: CloudPrintJobResponse, mesh: bytes, on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): - ResumableUpload(self._manager, upload_response.upload_url, upload_response.content_type, mesh, on_finished, - on_progress, on_error).start() + self._upload = MeshUploader(self._manager, print_job, mesh, on_finished, on_progress, on_error) + self._upload.start() # Requests a cluster to print the given print job. # \param cluster_id: The ID of the cluster. diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index 09677d5e48..88c2f8da1d 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -8,11 +8,13 @@ from typing import Dict, List, Optional, Set from PyQt5.QtCore import QObject, QUrl, pyqtProperty, pyqtSignal, pyqtSlot from UM import i18nCatalog +from UM.Backend.Backend import BackendState from UM.FileHandler.FileHandler import FileHandler from UM.Logger import Logger from UM.Message import Message from UM.Qt.Duration import Duration, DurationFormat from UM.Scene.SceneNode import SceneNode +from cura.CuraApplication import CuraApplication from cura.PrinterOutput.NetworkedPrinterOutputDevice import AuthState, NetworkedPrinterOutputDevice from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from plugins.UM3NetworkPrinting.src.Cloud.CloudOutputController import CloudOutputController @@ -92,6 +94,8 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): self._device_id = device_id self._account = api_client.account + CuraApplication.getInstance().getBackend().backendStateChange.connect(self._onBackendStateChange) + # We use the Cura Connect monitor tab to get most functionality right away. self._monitor_view_qml_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../resources/qml/MonitorStage.qml") @@ -116,6 +120,17 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # A set of the user's job IDs that have finished self._finished_jobs = set() # type: Set[str] + # Reference to the uploaded print job + self._mesh = None # type: Optional[bytes] + self._uploaded_print_job = None # type: Optional[CloudPrintJobResponse] + + def disconnect(self) -> None: + CuraApplication.getInstance().getBackend().backendStateChange.disconnect(self._onBackendStateChange) + + def _onBackendStateChange(self, _: BackendState) -> None: + self._mesh = None + self._uploaded_print_job = None + ## Gets the host name of this device @property def host_name(self) -> str: @@ -146,7 +161,16 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # Show an error message if we're already sending a job. if self._progress.visible: - self._onUploadError(T.BLOCKED_UPLOADING) + Message( + text = T.BLOCKED_UPLOADING, + title = T.ERROR, + lifetime = 10, + ).show() + return + + if self._uploaded_print_job: + # the mesh didn't change, let's not upload it again + self._api.requestPrint(self._device_id, self._uploaded_print_job.job_id, self._onPrintRequested) return # Indicate we have started sending a job. @@ -157,14 +181,15 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): Logger.log("e", "Missing file or mesh writer!") return self._onUploadError(T.COULD_NOT_EXPORT) - mesh_bytes = mesh_format.getBytes(nodes) + mesh = mesh_format.getBytes(nodes) + self._mesh = mesh request = CloudPrintJobUploadRequest( job_name = file_name, - file_size = len(mesh_bytes), + file_size = len(mesh), content_type = mesh_format.mime_type, ) - self._api.requestUpload(request, lambda response: self._onPrintJobCreated(mesh_bytes, response)) + self._api.requestUpload(request, self._onPrintJobCreated) ## Called when the network data should be updated. def _update(self) -> None: @@ -281,21 +306,22 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Uploads the mesh when the print job was registered with the cloud API. # \param mesh: The bytes to upload. # \param job_response: The response received from the cloud API. - def _onPrintJobCreated(self, mesh: bytes, job_response: CloudPrintJobResponse) -> None: + def _onPrintJobCreated(self, job_response: CloudPrintJobResponse) -> None: self._progress.show() - self._api.uploadMesh(job_response, mesh, lambda: self._onPrintJobUploaded(job_response.job_id), - self._progress.update, self._onUploadError) + self._uploaded_print_job = job_response + self._api.uploadMesh(job_response, self._mesh, self._onPrintJobUploaded, self._progress.update, + self._onUploadError) ## Requests the print to be sent to the printer when we finished uploading the mesh. - # \param job_id: The ID of the job. - def _onPrintJobUploaded(self, job_id: str) -> None: + def _onPrintJobUploaded(self) -> None: self._progress.update(100) - self._api.requestPrint(self._device_id, job_id, self._onPrintRequested) + self._api.requestPrint(self._device_id, self._uploaded_print_job.job_id, self._onPrintRequested) ## Displays the given message if uploading the mesh has failed # \param message: The message to display. def _onUploadError(self, message = None) -> None: self._progress.hide() + self._uploaded_print_job = None Message( text = message or T.UPLOAD_ERROR, title = T.ERROR, diff --git a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py b/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py similarity index 91% rename from plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py rename to plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py index 5e3bc9545e..4f0d6f2e81 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/ResumableUpload.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py @@ -6,9 +6,10 @@ from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManage from typing import Optional, Callable, Any, Tuple from UM.Logger import Logger +from src.Cloud.Models.CloudPrintJobResponse import CloudPrintJobResponse -class ResumableUpload: +class MeshUploader: MAX_RETRIES = 10 BYTES_PER_REQUEST = 256 * 1024 RETRY_HTTP_CODES = {500, 502, 503, 504} @@ -18,11 +19,10 @@ class ResumableUpload: # \param content_length: The total content length of the file, in bytes. # \param http_method: The HTTP method to be used, e.g. "POST" or "PUT". # \param timeout: The timeout for each chunk upload. Important: If None, no timeout is applied at all. - def __init__(self, manager: QNetworkAccessManager, url: str, content_type: str, data: bytes, + def __init__(self, manager: QNetworkAccessManager, print_job: CloudPrintJobResponse, data: bytes, on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): self._manager = manager - self._url = url - self._content_type = content_type + self._print_job = print_job self._data = data self._on_finished = on_finished @@ -34,17 +34,21 @@ class ResumableUpload: self._finished = False self._reply = None # type: Optional[QNetworkReply] + @property + def printJob(self): + return self._print_job + ## We override _createRequest in order to add the user credentials. # \param url: The URL to request # \param content_type: The type of the body contents. def _createRequest(self) -> QNetworkRequest: - request = QNetworkRequest(QUrl(self._url)) - request.setHeader(QNetworkRequest.ContentTypeHeader, self._content_type) + request = QNetworkRequest(QUrl(self._print_job.upload_url)) + request.setHeader(QNetworkRequest.ContentTypeHeader, self._print_job.content_type) first_byte, last_byte = self._chunkRange() content_range = "bytes {}-{}/{}".format(first_byte, last_byte - 1, len(self._data)) request.setRawHeader(b"Content-Range", content_range.encode()) - Logger.log("i", "Uploading %s to %s", content_range, self._url) + Logger.log("i", "Uploading %s to %s", content_range, self._print_job.upload_url) return request diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py index c391dc75dd..d31f59f85a 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py @@ -5,6 +5,7 @@ from unittest import TestCase from unittest.mock import patch, MagicMock from UM.Scene.SceneNode import SceneNode +from UM.Signal import Signal from cura.CuraApplication import CuraApplication from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from src.Cloud.CloudApiClient import CloudApiClient @@ -26,6 +27,9 @@ class TestCloudOutputDevice(TestCase): def setUp(self): super().setUp() self.app = CuraApplication.getInstance() + self.backend = MagicMock(backendStateChange = Signal()) + self.app.setBackend(self.backend) + self.network = NetworkManagerMock() self.account = MagicMock(isLoggedIn=True, accessToken="TestAccessToken") self.onError = MagicMock() From 9f4b7bd70362a3f7c77ebda0ee77d52ee68ca945 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 11:28:16 +0100 Subject: [PATCH 05/16] STAR-322: Improving logging and cluster connection --- cura/OAuth2/AuthorizationService.py | 3 +- .../src/Cloud/CloudApiClient.py | 29 ++++++++------ .../src/Cloud/CloudOutputDevice.py | 40 +++++++++++-------- .../src/Cloud/CloudOutputDeviceManager.py | 11 +++-- .../src/ClusterUM3OutputDevice.py | 2 +- .../src/ClusterUM3PrinterOutputController.py | 1 - 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/cura/OAuth2/AuthorizationService.py b/cura/OAuth2/AuthorizationService.py index 21dbbe8248..0c3074f3d5 100644 --- a/cura/OAuth2/AuthorizationService.py +++ b/cura/OAuth2/AuthorizationService.py @@ -52,7 +52,8 @@ class AuthorizationService: if not self._user_profile: # If no user profile was stored locally, we try to get it from JWT. self._user_profile = self._parseJWT() - if not self._user_profile: + + if not self._user_profile and self._auth_data: # If there is still no user profile from the JWT, we have to log in again. Logger.log("w", "The user profile could not be loaded. The user must log in again!") self.deleteAuthData() diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index b54c9e97d6..5fd14efc9c 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -2,6 +2,7 @@ # Cura is released under the terms of the LGPLv3 or higher. import json from json import JSONDecodeError +from time import time from typing import Callable, List, Type, TypeVar, Union, Optional, Tuple, Dict, Any from PyQt5.QtCore import QUrl @@ -38,6 +39,8 @@ class CloudApiClient: self._account = account self._on_error = on_error self._upload = None # type: Optional[MeshUploader] + # in order to avoid garbage collection we keep the callbacks in this list. + self._anti_gc_callbacks = [] # type: List[Callable[[QNetworkReply], None]] ## Gets the account used for the API. @property @@ -49,8 +52,7 @@ class CloudApiClient: def getClusters(self, on_finished: Callable[[List[CloudClusterResponse]], Any]) -> None: url = "{}/clusters".format(self.CLUSTER_API_ROOT) reply = self._manager.get(self._createEmptyRequest(url)) - callback = self._wrapCallback(reply, on_finished, CloudClusterResponse) - reply.finished.connect(callback) + self._addCallbacks(reply, on_finished, CloudClusterResponse) ## Retrieves the status of the given cluster. # \param cluster_id: The ID of the cluster. @@ -58,8 +60,7 @@ class CloudApiClient: def getClusterStatus(self, cluster_id: str, on_finished: Callable[[CloudClusterStatus], Any]) -> None: url = "{}/clusters/{}/status".format(self.CLUSTER_API_ROOT, cluster_id) reply = self._manager.get(self._createEmptyRequest(url)) - callback = self._wrapCallback(reply, on_finished, CloudClusterStatus) - reply.finished.connect(callback) + self._addCallbacks(reply, on_finished, CloudClusterStatus) ## Requests the cloud to register the upload of a print job mesh. # \param request: The request object. @@ -69,8 +70,7 @@ class CloudApiClient: url = "{}/jobs/upload".format(self.CURA_API_ROOT) body = json.dumps({"data": request.toDict()}) reply = self._manager.put(self._createEmptyRequest(url), body.encode()) - callback = self._wrapCallback(reply, on_finished, CloudPrintJobResponse) - reply.finished.connect(callback) + self._addCallbacks(reply, on_finished, CloudPrintJobResponse) ## Requests the cloud to register the upload of a print job mesh. # \param upload_response: The object received after requesting an upload with `self.requestUpload`. @@ -90,8 +90,7 @@ class CloudApiClient: def requestPrint(self, cluster_id: str, job_id: str, on_finished: Callable[[CloudPrintResponse], Any]) -> None: url = "{}/clusters/{}/print/{}".format(self.CLUSTER_API_ROOT, cluster_id, job_id) reply = self._manager.post(self._createEmptyRequest(url), b"") - callback = self._wrapCallback(reply, on_finished, CloudPrintResponse) - reply.finished.connect(callback) + self._addCallbacks(reply, on_finished, CloudPrintResponse) ## We override _createEmptyRequest in order to add the user credentials. # \param url: The URL to request @@ -116,9 +115,10 @@ class CloudApiClient: Logger.log("i", "Received a reply %s from %s with %s", status_code, reply.url().toString(), response) return status_code, json.loads(response) except (UnicodeDecodeError, JSONDecodeError, ValueError) as err: - error = {"code": type(err).__name__, "title": str(err), "http_code": str(status_code)} + error = CloudErrorObject(code=type(err).__name__, title=str(err), http_code=str(status_code), + id=str(time()), http_status="500") Logger.logException("e", "Could not parse the stardust response: %s", error) - return status_code, {"errors": [error]} + return status_code, {"errors": [error.toDict()]} ## The generic type variable used to document the methods below. Model = TypeVar("Model", bound=BaseModel) @@ -143,12 +143,15 @@ class CloudApiClient: # \param on_finished: The callback in case the response is successful. # \param model: The type of the model to convert the response to. It may either be a single record or a list. # \return: A function that can be passed to the - def _wrapCallback(self, + def _addCallbacks(self, reply: QNetworkReply, on_finished: Callable[[Union[Model, List[Model]]], Any], model: Type[Model], - ) -> Callable[[QNetworkReply], None]: + ) -> None: def parse() -> None: status_code, response = self._parseReply(reply) + self._anti_gc_callbacks.remove(parse) return self._parseModels(response, on_finished, model) - return parse + + self._anti_gc_callbacks.append(parse) + reply.finished.connect(parse) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index 88c2f8da1d..3890e7cee2 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -18,6 +18,7 @@ from cura.CuraApplication import CuraApplication from cura.PrinterOutput.NetworkedPrinterOutputDevice import AuthState, NetworkedPrinterOutputDevice from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from plugins.UM3NetworkPrinting.src.Cloud.CloudOutputController import CloudOutputController +from src.Cloud.Models.CloudClusterResponse import CloudClusterResponse from ..MeshFormatHandler import MeshFormatHandler from ..UM3PrintJobOutputModel import UM3PrintJobOutputModel from .CloudProgressMessage import CloudProgressMessage @@ -84,18 +85,15 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # \param api_client: The client that will run the API calls # \param device_id: The ID of the device (i.e. the cluster_id for the cloud API) # \param parent: The optional parent of this output device. - def __init__(self, api_client: CloudApiClient, device_id: str, host_name: str, parent: QObject = None) -> None: - super().__init__(device_id = device_id, address = "", properties = {}, parent = parent) + def __init__(self, api_client: CloudApiClient, cluster: CloudClusterResponse, parent: QObject = None) -> None: + super().__init__(device_id = cluster.cluster_id, address = "", properties = {}, parent = parent) self._api = api_client - self._host_name = host_name + self._cluster = cluster self._setInterfaceElements() - self._device_id = device_id self._account = api_client.account - CuraApplication.getInstance().getBackend().backendStateChange.connect(self._onBackendStateChange) - # We use the Cura Connect monitor tab to get most functionality right away. self._monitor_view_qml_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../resources/qml/MonitorStage.qml") @@ -124,7 +122,14 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): self._mesh = None # type: Optional[bytes] self._uploaded_print_job = None # type: Optional[CloudPrintJobResponse] + def connect(self) -> None: + super().connect() + Logger.log("i", "Connected to cluster %s", self.key) + CuraApplication.getInstance().getBackend().backendStateChange.connect(self._onBackendStateChange) + def disconnect(self) -> None: + super().disconnect() + Logger.log("i", "Disconnected to cluster %s", self.key) CuraApplication.getInstance().getBackend().backendStateChange.disconnect(self._onBackendStateChange) def _onBackendStateChange(self, _: BackendState) -> None: @@ -133,19 +138,19 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Gets the host name of this device @property - def host_name(self) -> str: - return self._host_name + def clusterData(self) -> CloudClusterResponse: + return self._cluster ## Updates the host name of the output device - @host_name.setter - def host_name(self, value: str) -> None: - self._host_name = value + @clusterData.setter + def clusterData(self, value: CloudClusterResponse) -> None: + self._cluster = value ## Checks whether the given network key is found in the cloud's host name def matchesNetworkKey(self, network_key: str) -> bool: # A network key looks like "ultimakersystem-aabbccdd0011._ultimaker._tcp.local." # the host name should then be "ultimakersystem-aabbccdd0011" - return network_key.startswith(self._host_name) + return network_key.startswith(self.clusterData.host_name) ## Set all the interface elements and texts for this output device. def _setInterfaceElements(self) -> None: @@ -170,7 +175,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): if self._uploaded_print_job: # the mesh didn't change, let's not upload it again - self._api.requestPrint(self._device_id, self._uploaded_print_job.job_id, self._onPrintRequested) + self._api.requestPrint(self.key, self._uploaded_print_job.job_id, self._onPrintRequested) return # Indicate we have started sending a job. @@ -194,12 +199,15 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Called when the network data should be updated. def _update(self) -> None: super()._update() - if self._last_response_time and time() - self._last_response_time < self.CHECK_CLUSTER_INTERVAL: + if self._last_request_time and time() - self._last_request_time < self.CHECK_CLUSTER_INTERVAL: + Logger.log("i", "Not updating: %s - %s < %s", time(), self._last_request_time, self.CHECK_CLUSTER_INTERVAL) return # avoid calling the cloud too often + Logger.log("i", "Updating: %s - %s >= %s", time(), self._last_request_time, self.CHECK_CLUSTER_INTERVAL) if self._account.isLoggedIn: self.setAuthenticationState(AuthState.Authenticated) - self._api.getClusterStatus(self._device_id, self._onStatusCallFinished) + self._last_request_time = time() + self._api.getClusterStatus(self.key, self._onStatusCallFinished) else: self.setAuthenticationState(AuthState.NotAuthenticated) @@ -315,7 +323,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Requests the print to be sent to the printer when we finished uploading the mesh. def _onPrintJobUploaded(self) -> None: self._progress.update(100) - self._api.requestPrint(self._device_id, self._uploaded_print_job.job_id, self._onPrintRequested) + self._api.requestPrint(self.key, self._uploaded_print_job.job_id, self._onPrintRequested) ## Displays the given message if uploading the mesh has failed # \param message: The message to display. diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py index af80907f01..29c60fd14a 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py @@ -73,7 +73,8 @@ class CloudOutputDeviceManager: removed_devices, added_clusters, updates = findChanges(self._remote_clusters, online_clusters) - Logger.log("i", "Parsed remote clusters to %s", online_clusters) + Logger.log("i", "Parsed remote clusters to %s", [cluster.toDict() for cluster in online_clusters.values()]) + Logger.log("i", "Removed: %s, added: %s, updates: %s", len(removed_devices), len(added_clusters), len(updates)) # Remove output devices that are gone for removed_cluster in removed_devices: @@ -86,12 +87,12 @@ class CloudOutputDeviceManager: # Add an output device for each new remote cluster. # We only add when is_online as we don't want the option in the drop down if the cluster is not online. for added_cluster in added_clusters: - device = CloudOutputDevice(self._api, added_cluster.cluster_id, added_cluster.host_name) + device = CloudOutputDevice(self._api, added_cluster) self._output_device_manager.addOutputDevice(device) self._remote_clusters[added_cluster.cluster_id] = device for device, cluster in updates: - device.host_name = cluster.host_name + device.clusterData = cluster self._connectToActiveMachine() @@ -99,6 +100,7 @@ class CloudOutputDeviceManager: def _connectToActiveMachine(self) -> None: active_machine = CuraApplication.getInstance().getGlobalContainerStack() if not active_machine: + Logger.log("i", "no active machine") return # Check if the stored cluster_id for the active machine is in our list of remote clusters. @@ -107,6 +109,7 @@ class CloudOutputDeviceManager: device = self._remote_clusters[stored_cluster_id] if not device.isConnected(): device.connect() + Logger.log("i", "Device connected by metadata %s", stored_cluster_id) else: self._connectByNetworkKey(active_machine) @@ -122,6 +125,8 @@ class CloudOutputDeviceManager: active_machine.setMetaDataEntry(self.META_CLUSTER_ID, device.key) device.connect() + Logger.log("i", "Found cluster %s with network key %s", device, local_network_key) + ## Handles an API error received from the cloud. # \param errors: The errors received def _onApiError(self, errors: List[CloudErrorObject]) -> None: diff --git a/plugins/UM3NetworkPrinting/src/ClusterUM3OutputDevice.py b/plugins/UM3NetworkPrinting/src/ClusterUM3OutputDevice.py index 96fee0d96d..1896ffac9c 100644 --- a/plugins/UM3NetworkPrinting/src/ClusterUM3OutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/ClusterUM3OutputDevice.py @@ -390,10 +390,10 @@ class ClusterUM3OutputDevice(NetworkedPrinterOutputDevice): ## Called when the connection to the cluster changes. def connect(self) -> None: - pass # TODO: uncomment this once cloud implementation works for testing # super().connect() # self.sendMaterialProfiles() + pass def _onGetPreviewImageFinished(self, reply: QNetworkReply) -> None: reply_url = reply.url().toString() diff --git a/plugins/UM3NetworkPrinting/src/ClusterUM3PrinterOutputController.py b/plugins/UM3NetworkPrinting/src/ClusterUM3PrinterOutputController.py index fcced0b883..fc6798386a 100644 --- a/plugins/UM3NetworkPrinting/src/ClusterUM3PrinterOutputController.py +++ b/plugins/UM3NetworkPrinting/src/ClusterUM3PrinterOutputController.py @@ -18,4 +18,3 @@ class ClusterUM3PrinterOutputController(PrinterOutputController): def setJobState(self, job: "PrintJobOutputModel", state: str): data = "{\"action\": \"%s\"}" % state self._output_device.put("print_jobs/%s/action" % job.key, data, on_finished=None) - From 9eb743bcb8b1619769f0c6679bb3378b9c813231 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 12:01:10 +0100 Subject: [PATCH 06/16] STAR-322: Making mypy happy --- cura/NetworkClient.py | 57 +++++++++++-------- cura/Settings/SimpleModeSettingsManager.py | 5 +- .../src/Cloud/CloudApiClient.py | 16 ++++-- .../src/Cloud/CloudOutputDevice.py | 13 +++-- .../src/Cloud/MeshUploader.py | 19 ++++--- 5 files changed, 64 insertions(+), 46 deletions(-) diff --git a/cura/NetworkClient.py b/cura/NetworkClient.py index 878158542a..fabaaed95e 100644 --- a/cura/NetworkClient.py +++ b/cura/NetworkClient.py @@ -1,7 +1,7 @@ # Copyright (c) 2018 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. from time import time -from typing import Optional, Dict, Callable, List, Union +from typing import Optional, Dict, Callable, List, Union, cast from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QHttpMultiPart, QNetworkRequest, QHttpPart, \ @@ -19,7 +19,7 @@ class NetworkClient: super().__init__() # Network manager instance to use for this client. - self._manager = None # type: Optional[QNetworkAccessManager] + self.__manager = None # type: Optional[QNetworkAccessManager] # Timings. self._last_manager_create_time = None # type: Optional[float] @@ -34,20 +34,26 @@ class NetworkClient: # HTTP which uses them. We hold references to these QHttpMultiPart objects here. self._kept_alive_multiparts = {} # type: Dict[QNetworkReply, QHttpMultiPart] + # in order to avoid garbage collection we keep the callbacks in this list. + self._anti_gc_callbacks = [] # type: List[Callable[[], None]] + ## Creates a network manager if needed, with all the required properties and event bindings. def start(self) -> None: - if self._manager: - return - self._manager = QNetworkAccessManager() - self._last_manager_create_time = time() - self._manager.authenticationRequired.connect(self._onAuthenticationRequired) + if not self.__manager: + self.__manager = QNetworkAccessManager() + self._last_manager_create_time = time() + self.__manager.authenticationRequired.connect(self._onAuthenticationRequired) ## Destroys the network manager and event bindings. def stop(self) -> None: - if not self._manager: - return - self._manager.authenticationRequired.disconnect(self._onAuthenticationRequired) - self._manager = None + if self.__manager: + self.__manager.authenticationRequired.disconnect(self._onAuthenticationRequired) + self.__manager = None + + @property + def _manager(self) -> QNetworkAccessManager: + self.start() + return cast(QNetworkAccessManager, self.__manager) ## Create a new empty network request. # Automatically adds the required HTTP headers. @@ -94,13 +100,13 @@ class NetworkClient: return self._createFormPart(content_header, data, content_type) ## Sends a put request to the given path. - # url: The path after the API prefix. - # data: The data to be sent in the body - # content_type: The content type of the body data. - # on_finished: The function to call when the response is received. - # on_progress: The function to call when the progress changes. Parameters are bytes_sent / bytes_total. - def put(self, url: str, data: Union[str, bytes], content_type: Optional[str] = None, - on_finished: Optional[Callable[[QNetworkReply], None]] = None, + # \param url: The path after the API prefix. + # \param data: The data to be sent in the body + # \param content_type: The content type of the body data. + # \param on_finished: The function to call when the response is received. + # \param on_progress: The function to call when the progress changes. Parameters are bytes_sent / bytes_total. + def put(self, url: str, data: Union[str, bytes], content_type: str, + on_finished: Callable[[QNetworkReply], None], on_progress: Optional[Callable[[int, int], None]] = None) -> None: request = self._createEmptyRequest(url, content_type = content_type) @@ -114,7 +120,7 @@ class NetworkClient: ## Sends a delete request to the given path. # url: The path after the API prefix. # on_finished: The function to be call when the response is received. - def delete(self, url: str, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: + def delete(self, url: str, on_finished: Callable[[QNetworkReply], None]) -> None: request = self._createEmptyRequest(url) reply = self._manager.deleteResource(request) callback = self._createCallback(reply, on_finished) @@ -123,7 +129,7 @@ class NetworkClient: ## Sends a get request to the given path. # \param url: The path after the API prefix. # \param on_finished: The function to be call when the response is received. - def get(self, url: str, on_finished: Optional[Callable[[QNetworkReply], None]]) -> None: + def get(self, url: str, on_finished: Callable[[QNetworkReply], None]) -> None: request = self._createEmptyRequest(url) reply = self._manager.get(request) callback = self._createCallback(reply, on_finished) @@ -135,7 +141,7 @@ class NetworkClient: # \param on_finished: The function to call when the response is received. # \param on_progress: The function to call when the progress changes. Parameters are bytes_sent / bytes_total. def post(self, url: str, data: Union[str, bytes], - on_finished: Optional[Callable[[QNetworkReply], None]], + on_finished: Callable[[QNetworkReply], None], on_progress: Optional[Callable[[int, int], None]] = None) -> None: request = self._createEmptyRequest(url) @@ -180,6 +186,9 @@ class NetworkClient: return reply - @staticmethod - def _createCallback(reply: QNetworkReply, on_finished: Optional[Callable[[QNetworkReply], None]] = None): - return lambda: on_finished(reply) + def _createCallback(self, reply: QNetworkReply, on_finished: Callable[[QNetworkReply], None]) -> Callable[[], None]: + def callback(): + on_finished(reply) + self._anti_gc_callbacks.remove(callback) + self._anti_gc_callbacks.append(callback) + return callback diff --git a/cura/Settings/SimpleModeSettingsManager.py b/cura/Settings/SimpleModeSettingsManager.py index 210a5794d4..b22aea15ea 100644 --- a/cura/Settings/SimpleModeSettingsManager.py +++ b/cura/Settings/SimpleModeSettingsManager.py @@ -1,5 +1,6 @@ # Copyright (c) 2017 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. +from typing import Set from PyQt5.QtCore import QObject, pyqtSignal, pyqtProperty, pyqtSlot @@ -63,10 +64,10 @@ class SimpleModeSettingsManager(QObject): @pyqtSlot() def updateIsProfileUserCreated(self) -> None: - quality_changes_keys = set() + quality_changes_keys = set() # type: Set[str] if not self._machine_manager.activeMachine: - return False + return global_stack = self._machine_manager.activeMachine diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index 5fd14efc9c..d58def4545 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -3,7 +3,7 @@ import json from json import JSONDecodeError from time import time -from typing import Callable, List, Type, TypeVar, Union, Optional, Tuple, Dict, Any +from typing import Callable, List, Type, TypeVar, Union, Optional, Tuple, Dict, Any, cast from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManager @@ -40,7 +40,7 @@ class CloudApiClient: self._on_error = on_error self._upload = None # type: Optional[MeshUploader] # in order to avoid garbage collection we keep the callbacks in this list. - self._anti_gc_callbacks = [] # type: List[Callable[[QNetworkReply], None]] + self._anti_gc_callbacks = [] # type: List[Callable[[], None]] ## Gets the account used for the API. @property @@ -128,12 +128,16 @@ class CloudApiClient: # \param on_finished: The callback in case the response is successful. # \param model_class: The type of the model to convert the response to. It may either be a single record or a list. def _parseModels(self, response: Dict[str, Any], - on_finished: Callable[[Union[Model, List[Model]]], Any], + on_finished: Union[Callable[[Model], Any], Callable[[List[Model]], Any]], model_class: Type[Model]) -> None: if "data" in response: data = response["data"] - result = [model_class(**c) for c in data] if isinstance(data, list) else model_class(**data) - on_finished(result) + if isinstance(data, list): + results = [model_class(**c) for c in data] # type: List[CloudApiClient.Model] + cast(Callable[[List[CloudApiClient.Model]], Any], on_finished)(results) + else: + result = model_class(**data) # type: CloudApiClient.Model + cast(Callable[[CloudApiClient.Model], Any], on_finished)(result) elif "errors" in response: self._on_error([CloudErrorObject(**error) for error in response["errors"]]) else: @@ -145,7 +149,7 @@ class CloudApiClient: # \return: A function that can be passed to the def _addCallbacks(self, reply: QNetworkReply, - on_finished: Callable[[Union[Model, List[Model]]], Any], + on_finished: Union[Callable[[Model], Any], Callable[[List[Model]], Any]], model: Type[Model], ) -> None: def parse() -> None: diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index 3890e7cee2..54f0dc20b6 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -3,7 +3,7 @@ import os from time import time -from typing import Dict, List, Optional, Set +from typing import Dict, List, Optional, Set, cast from PyQt5.QtCore import QObject, QUrl, pyqtProperty, pyqtSignal, pyqtSlot @@ -161,7 +161,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): self.setConnectionText(T.CONNECTED_VIA_CLOUD) ## Called when Cura requests an output device to receive a (G-code) file. - def requestWrite(self, nodes: List[SceneNode], file_name: Optional[str] = None, limit_mime_types: bool = False, + def requestWrite(self, nodes: List[SceneNode], file_name: Optional[str] = None, limit_mimetypes: bool = False, file_handler: Optional[FileHandler] = None, **kwargs: str) -> None: # Show an error message if we're already sending a job. @@ -190,7 +190,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): self._mesh = mesh request = CloudPrintJobUploadRequest( - job_name = file_name, + job_name = file_name or mesh_format.file_extension, file_size = len(mesh), content_type = mesh_format.mime_type, ) @@ -317,13 +317,14 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): def _onPrintJobCreated(self, job_response: CloudPrintJobResponse) -> None: self._progress.show() self._uploaded_print_job = job_response - self._api.uploadMesh(job_response, self._mesh, self._onPrintJobUploaded, self._progress.update, - self._onUploadError) + mesh = cast(bytes, self._mesh) + self._api.uploadMesh(job_response, mesh, self._onPrintJobUploaded, self._progress.update, self._onUploadError) ## Requests the print to be sent to the printer when we finished uploading the mesh. def _onPrintJobUploaded(self) -> None: self._progress.update(100) - self._api.requestPrint(self.key, self._uploaded_print_job.job_id, self._onPrintRequested) + print_job = cast(CloudPrintJobResponse, self._uploaded_print_job) + self._api.requestPrint(self.key, print_job.job_id, self._onPrintRequested) ## Displays the given message if uploading the mesh has failed # \param message: The message to display. diff --git a/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py b/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py index 4f0d6f2e81..f0360b83e3 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py @@ -3,7 +3,7 @@ # -*- coding: utf-8 -*- from PyQt5.QtCore import QUrl from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply, QNetworkAccessManager -from typing import Optional, Callable, Any, Tuple +from typing import Optional, Callable, Any, Tuple, cast from UM.Logger import Logger from src.Cloud.Models.CloudPrintJobResponse import CloudPrintJobResponse @@ -20,7 +20,8 @@ class MeshUploader: # \param http_method: The HTTP method to be used, e.g. "POST" or "PUT". # \param timeout: The timeout for each chunk upload. Important: If None, no timeout is applied at all. def __init__(self, manager: QNetworkAccessManager, print_job: CloudPrintJobResponse, data: bytes, - on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): + on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any] + ) -> None: self._manager = manager self._print_job = print_job self._data = data @@ -85,20 +86,22 @@ class MeshUploader: self._on_progress(int((self._sent_bytes + bytes_sent) / len(self._data) * 100)) def _errorCallback(self) -> None: - body = bytes(self._reply.readAll()).decode() + reply = cast(QNetworkReply, self._reply) + body = bytes(reply.readAll()).decode() Logger.log("e", "Received error while uploading: %s", body) self.stop() self._on_error() def _finishedCallback(self) -> None: + reply = cast(QNetworkReply, self._reply) Logger.log("i", "Finished callback %s %s", - self._reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), self._reply.url().toString()) + reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url().toString()) - status_code = self._reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) + status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if self._retries < self.MAX_RETRIES and status_code in self.RETRY_HTTP_CODES: self._retries += 1 - Logger.log("i", "Retrying %s/%s request %s", self._retries, self.MAX_RETRIES, self._reply.url().toString()) + Logger.log("i", "Retrying %s/%s request %s", self._retries, self.MAX_RETRIES, reply.url().toString()) self._uploadChunk() return @@ -106,9 +109,9 @@ class MeshUploader: self._errorCallback() return - body = bytes(self._reply.readAll()).decode() + body = bytes(reply.readAll()).decode() Logger.log("w", "status_code: %s, Headers: %s, body: %s", status_code, - [bytes(header).decode() for header in self._reply.rawHeaderList()], body) + [bytes(header).decode() for header in reply.rawHeaderList()], body) first_byte, last_byte = self._chunkRange() self._sent_bytes += last_byte - first_byte From 87c1392173ce05785f9e4c981bb1cae86dcbacce Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 12:05:25 +0100 Subject: [PATCH 07/16] STAR-322: Fixing tests --- .../tests/Cloud/TestCloudOutputDevice.py | 10 +++++++--- .../tests/Cloud/TestCloudOutputDeviceManager.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py index d31f59f85a..c42a208ece 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py @@ -10,6 +10,7 @@ from cura.CuraApplication import CuraApplication from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from src.Cloud.CloudApiClient import CloudApiClient from src.Cloud.CloudOutputDevice import CloudOutputDevice +from src.Cloud.Models.CloudClusterResponse import CloudClusterResponse from tests.Cloud.Fixtures import readFixture, parseFixture from .NetworkManagerMock import NetworkManagerMock @@ -18,6 +19,7 @@ class TestCloudOutputDevice(TestCase): CLUSTER_ID = "RIZ6cZbWA_Ua7RZVJhrdVfVpf0z-MqaSHQE4v8aRTtYq" JOB_ID = "ABCDefGHIjKlMNOpQrSTUvYxWZ0-1234567890abcDE=" HOST_NAME = "ultimakersystem-ccbdd30044ec" + HOST_GUID = "e90ae0ac-1257-4403-91ee-a44c9b7e8050" BASE_URL = "https://api-staging.ultimaker.com" STATUS_URL = "{}/connect/v1/clusters/{}/status".format(BASE_URL, CLUSTER_ID) @@ -29,13 +31,15 @@ class TestCloudOutputDevice(TestCase): self.app = CuraApplication.getInstance() self.backend = MagicMock(backendStateChange = Signal()) self.app.setBackend(self.backend) + self.cluster = CloudClusterResponse(self.CLUSTER_ID, self.HOST_GUID, self.HOST_NAME, is_online=True, + status="active") self.network = NetworkManagerMock() self.account = MagicMock(isLoggedIn=True, accessToken="TestAccessToken") self.onError = MagicMock() with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): self._api = CloudApiClient(self.account, self.onError) - self.device = CloudOutputDevice(self._api, self.CLUSTER_ID, self.HOST_NAME) + self.device = CloudOutputDevice(self._api, self.cluster) self.cluster_status = parseFixture("getClusterStatusResponse") self.network.prepareReply("GET", self.STATUS_URL, 200, readFixture("getClusterStatusResponse")) @@ -81,7 +85,7 @@ class TestCloudOutputDevice(TestCase): self.cluster_status["data"]["print_jobs"].clear() self.network.prepareReply("GET", self.STATUS_URL, 200, self.cluster_status) - self.device._last_response_time = None + self.device._last_request_time = None self.device._update() self.network.flushReplies() self.assertEqual([], self.device.printJobs) @@ -94,7 +98,7 @@ class TestCloudOutputDevice(TestCase): self.cluster_status["data"]["printers"].clear() self.network.prepareReply("GET", self.STATUS_URL, 200, self.cluster_status) - self.device._last_response_time = None + self.device._last_request_time = None self.device._update() self.network.flushReplies() self.assertEqual([], self.device.printers) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py index 80dd2c7990..f98e954274 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py @@ -41,7 +41,7 @@ class TestCloudOutputDeviceManager(TestCase): clusters = self.clusters_response.get("data", []) self.assertEqual([CloudOutputDevice] * len(clusters), [type(d) for d in devices]) self.assertEqual({cluster["cluster_id"] for cluster in clusters}, {device.key for device in devices}) - self.assertEqual({cluster["host_name"] for cluster in clusters}, {device.host_name for device in devices}) + self.assertEqual(clusters, [device.clusterData.toDict() for device in devices]) for device in clusters: device_manager.getOutputDevice(device["cluster_id"]).close() From 34f56b046aeb562c8d8e8d52a1dd89d35474c176 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 13:13:44 +0100 Subject: [PATCH 08/16] STAR-322: Review comments --- cura/NetworkClient.py | 194 ------------------ .../src/Cloud/CloudOutputDevice.py | 2 +- 2 files changed, 1 insertion(+), 195 deletions(-) delete mode 100644 cura/NetworkClient.py diff --git a/cura/NetworkClient.py b/cura/NetworkClient.py deleted file mode 100644 index fabaaed95e..0000000000 --- a/cura/NetworkClient.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2018 Ultimaker B.V. -# Cura is released under the terms of the LGPLv3 or higher. -from time import time -from typing import Optional, Dict, Callable, List, Union, cast - -from PyQt5.QtCore import QUrl -from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QHttpMultiPart, QNetworkRequest, QHttpPart, \ - QAuthenticator - -from UM.Application import Application -from UM.Logger import Logger - - -## Abstraction of QNetworkAccessManager for easier networking in Cura. -# This was originally part of NetworkedPrinterOutputDevice but was moved out for re-use in other classes. -class NetworkClient: - - def __init__(self) -> None: - super().__init__() - - # Network manager instance to use for this client. - self.__manager = None # type: Optional[QNetworkAccessManager] - - # Timings. - self._last_manager_create_time = None # type: Optional[float] - self._last_response_time = None # type: Optional[float] - self._last_request_time = None # type: Optional[float] - - # The user agent of Cura. - application = Application.getInstance() - self._user_agent = "%s/%s " % (application.getApplicationName(), application.getVersion()) - - # QHttpMultiPart objects need to be kept alive and not garbage collected during the - # HTTP which uses them. We hold references to these QHttpMultiPart objects here. - self._kept_alive_multiparts = {} # type: Dict[QNetworkReply, QHttpMultiPart] - - # in order to avoid garbage collection we keep the callbacks in this list. - self._anti_gc_callbacks = [] # type: List[Callable[[], None]] - - ## Creates a network manager if needed, with all the required properties and event bindings. - def start(self) -> None: - if not self.__manager: - self.__manager = QNetworkAccessManager() - self._last_manager_create_time = time() - self.__manager.authenticationRequired.connect(self._onAuthenticationRequired) - - ## Destroys the network manager and event bindings. - def stop(self) -> None: - if self.__manager: - self.__manager.authenticationRequired.disconnect(self._onAuthenticationRequired) - self.__manager = None - - @property - def _manager(self) -> QNetworkAccessManager: - self.start() - return cast(QNetworkAccessManager, self.__manager) - - ## Create a new empty network request. - # Automatically adds the required HTTP headers. - # \param url: The URL to request - # \param content_type: The type of the body contents. - def _createEmptyRequest(self, url: str, content_type: Optional[str] = "application/json") -> QNetworkRequest: - if not self._manager: - self.start() # make sure the manager is created - request = QNetworkRequest(QUrl(url)) - if content_type: - request.setHeader(QNetworkRequest.ContentTypeHeader, content_type) - request.setHeader(QNetworkRequest.UserAgentHeader, self._user_agent) - self._last_request_time = time() - return request - - ## Removes all cached Multi-Part items. - def _clearCachedMultiPart(self, reply: QNetworkReply) -> None: - if reply in self._kept_alive_multiparts: - del self._kept_alive_multiparts[reply] - - ## Callback for when the network manager detects that authentication is required but was not given. - @staticmethod - def _onAuthenticationRequired(reply: QNetworkReply, authenticator: QAuthenticator) -> None: - Logger.log("w", "Request to {} required authentication but was not given".format(reply.url().toString())) - - ## Add a part to a Multi-Part form. - @staticmethod - def _createFormPart(content_header: str, data: bytes, content_type: Optional[str] = None) -> QHttpPart: - part = QHttpPart() - - if not content_header.startswith("form-data;"): - content_header = "form_data; " + content_header - - part.setHeader(QNetworkRequest.ContentDispositionHeader, content_header) - - if content_type is not None: - part.setHeader(QNetworkRequest.ContentTypeHeader, content_type) - - part.setBody(data) - return part - - ## Public version of _createFormPart. Both are needed for backward compatibility with 3rd party plugins. - def createFormPart(self, content_header: str, data: bytes, content_type: Optional[str] = None) -> QHttpPart: - return self._createFormPart(content_header, data, content_type) - - ## Sends a put request to the given path. - # \param url: The path after the API prefix. - # \param data: The data to be sent in the body - # \param content_type: The content type of the body data. - # \param on_finished: The function to call when the response is received. - # \param on_progress: The function to call when the progress changes. Parameters are bytes_sent / bytes_total. - def put(self, url: str, data: Union[str, bytes], content_type: str, - on_finished: Callable[[QNetworkReply], None], - on_progress: Optional[Callable[[int, int], None]] = None) -> None: - request = self._createEmptyRequest(url, content_type = content_type) - - body = data if isinstance(data, bytes) else data.encode() # type: bytes - reply = self._manager.put(request, body) - callback = self._createCallback(reply, on_finished) - reply.finished.connect(callback) - if on_progress is not None: - reply.uploadProgress.connect(on_progress) - - ## Sends a delete request to the given path. - # url: The path after the API prefix. - # on_finished: The function to be call when the response is received. - def delete(self, url: str, on_finished: Callable[[QNetworkReply], None]) -> None: - request = self._createEmptyRequest(url) - reply = self._manager.deleteResource(request) - callback = self._createCallback(reply, on_finished) - reply.finished.connect(callback) - - ## Sends a get request to the given path. - # \param url: The path after the API prefix. - # \param on_finished: The function to be call when the response is received. - def get(self, url: str, on_finished: Callable[[QNetworkReply], None]) -> None: - request = self._createEmptyRequest(url) - reply = self._manager.get(request) - callback = self._createCallback(reply, on_finished) - reply.finished.connect(callback) - - ## Sends a post request to the given path. - # \param url: The path after the API prefix. - # \param data: The data to be sent in the body - # \param on_finished: The function to call when the response is received. - # \param on_progress: The function to call when the progress changes. Parameters are bytes_sent / bytes_total. - def post(self, url: str, data: Union[str, bytes], - on_finished: Callable[[QNetworkReply], None], - on_progress: Optional[Callable[[int, int], None]] = None) -> None: - request = self._createEmptyRequest(url) - - body = data if isinstance(data, bytes) else data.encode() # type: bytes - reply = self._manager.post(request, body) - callback = self._createCallback(reply, on_finished) - reply.finished.connect(callback) - if on_progress is not None: - reply.uploadProgress.connect(on_progress) - - ## Does a POST request with form data to the given URL. - def postForm(self, url: str, header_data: str, body_data: bytes, - on_finished: Optional[Callable[[QNetworkReply], None]], - on_progress: Optional[Callable[[int, int], None]] = None) -> None: - post_part = QHttpPart() - post_part.setHeader(QNetworkRequest.ContentDispositionHeader, header_data) - post_part.setBody(body_data) - self.postFormWithParts(url, [post_part], on_finished, on_progress) - - ## Does a POST request with form parts to the given URL. - def postFormWithParts(self, target: str, parts: List[QHttpPart], - on_finished: Optional[Callable[[QNetworkReply], None]], - on_progress: Optional[Callable[[int, int], None]] = None) -> Optional[QNetworkReply]: - request = self._createEmptyRequest(target, content_type = None) - multi_post_part = QHttpMultiPart(QHttpMultiPart.FormDataType) - - for part in parts: - multi_post_part.append(part) - - reply = self._manager.post(request, multi_post_part) - - def callback(): - on_finished(reply) - self._clearCachedMultiPart(reply) - - reply.finished.connect(callback) - - self._kept_alive_multiparts[reply] = multi_post_part - - if on_progress is not None: - reply.uploadProgress.connect(on_progress) - - return reply - - def _createCallback(self, reply: QNetworkReply, on_finished: Callable[[QNetworkReply], None]) -> Callable[[], None]: - def callback(): - on_finished(reply) - self._anti_gc_callbacks.remove(callback) - self._anti_gc_callbacks.append(callback) - return callback diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index 54f0dc20b6..f29e29c40b 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -18,11 +18,11 @@ from cura.CuraApplication import CuraApplication from cura.PrinterOutput.NetworkedPrinterOutputDevice import AuthState, NetworkedPrinterOutputDevice from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from plugins.UM3NetworkPrinting.src.Cloud.CloudOutputController import CloudOutputController -from src.Cloud.Models.CloudClusterResponse import CloudClusterResponse from ..MeshFormatHandler import MeshFormatHandler from ..UM3PrintJobOutputModel import UM3PrintJobOutputModel from .CloudProgressMessage import CloudProgressMessage from .CloudApiClient import CloudApiClient +from .Models.CloudClusterResponse import CloudClusterResponse from .Models.CloudClusterStatus import CloudClusterStatus from .Models.CloudPrintJobUploadRequest import CloudPrintJobUploadRequest from .Models.CloudPrintResponse import CloudPrintResponse From a1cb09f73fe4403090c4857c35910ba92e74e411 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 13:24:07 +0100 Subject: [PATCH 09/16] STAR-322: Sorting clusters --- plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py | 2 ++ .../UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py | 2 ++ .../tests/Cloud/TestCloudOutputDeviceManager.py | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py index d4044726a3..c7d58cea78 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py @@ -18,6 +18,8 @@ from .NetworkManagerMock import NetworkManagerMock class TestCloudApiClient(TestCase): + maxDiff = None + def _errorHandler(self, errors: List[CloudErrorObject]): raise Exception("Received unexpected error: {}".format(errors)) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py index c42a208ece..fded79e15b 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py @@ -16,6 +16,8 @@ from .NetworkManagerMock import NetworkManagerMock class TestCloudOutputDevice(TestCase): + maxDiff = None + CLUSTER_ID = "RIZ6cZbWA_Ua7RZVJhrdVfVpf0z-MqaSHQE4v8aRTtYq" JOB_ID = "ABCDefGHIjKlMNOpQrSTUvYxWZ0-1234567890abcDE=" HOST_NAME = "ultimakersystem-ccbdd30044ec" diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py index f98e954274..f62d92e9db 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py @@ -11,6 +11,8 @@ from .NetworkManagerMock import NetworkManagerMock class TestCloudOutputDeviceManager(TestCase): + maxDiff = None + URL = "https://api-staging.ultimaker.com/connect/v1/clusters" def setUp(self): @@ -41,7 +43,8 @@ class TestCloudOutputDeviceManager(TestCase): clusters = self.clusters_response.get("data", []) self.assertEqual([CloudOutputDevice] * len(clusters), [type(d) for d in devices]) self.assertEqual({cluster["cluster_id"] for cluster in clusters}, {device.key for device in devices}) - self.assertEqual(clusters, [device.clusterData.toDict() for device in devices]) + self.assertEqual(clusters, sorted((device.clusterData.toDict() for device in devices), + key=lambda device_dict: device_dict["host_version"])) for device in clusters: device_manager.getOutputDevice(device["cluster_id"]).close() From b6f90f1ab2caa1be9a1974f680970997dd8e72b3 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 13:29:20 +0100 Subject: [PATCH 10/16] STAR-322: Using cura constants --- .../UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py | 9 +++++---- .../tests/Cloud/TestCloudOutputDevice.py | 8 ++++---- .../tests/Cloud/TestCloudOutputDeviceManager.py | 3 ++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py index c7d58cea78..a09894deb0 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py @@ -7,6 +7,7 @@ from unittest import TestCase from unittest.mock import patch, MagicMock from cura.CuraApplication import CuraApplication +from cura.CuraConstants import CuraCloudAPIRoot from src.Cloud.CloudApiClient import CloudApiClient from src.Cloud.Models.CloudClusterResponse import CloudClusterResponse from src.Cloud.Models.CloudClusterStatus import CloudClusterStatus @@ -39,7 +40,7 @@ class TestCloudApiClient(TestCase): response = readFixture("getClusters") data = parseFixture("getClusters")["data"] - self.network.prepareReply("GET", "https://api-staging.ultimaker.com/connect/v1/clusters", 200, response) + self.network.prepareReply("GET", CuraCloudAPIRoot + "/connect/v1/clusters", 200, response) # the callback is a function that adds the result of the call to getClusters to the result list self.api.getClusters(lambda clusters: result.extend(clusters)) @@ -54,7 +55,7 @@ class TestCloudApiClient(TestCase): data = parseFixture("getClusterStatusResponse")["data"] self.network.prepareReply("GET", - "https://api-staging.ultimaker.com/connect/v1/clusters/R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC/status", + CuraCloudAPIRoot + "/connect/v1/clusters/R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC/status", 200, response ) self.api.getClusterStatus("R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC", lambda s: result.append(s)) @@ -69,7 +70,7 @@ class TestCloudApiClient(TestCase): response = readFixture("putJobUploadResponse") - self.network.prepareReply("PUT", "https://api-staging.ultimaker.com/cura/v1/jobs/upload", 200, response) + self.network.prepareReply("PUT", CuraCloudAPIRoot + "/cura/v1/jobs/upload", 200, response) request = CloudPrintJobUploadRequest(job_name = "job name", file_size = 143234, content_type = "text/plain") self.api.requestUpload(request, lambda r: results.append(r)) self.network.flushReplies() @@ -108,7 +109,7 @@ class TestCloudApiClient(TestCase): job_id = "ABCDefGHIjKlMNOpQrSTUvYxWZ0-1234567890abcDE=" self.network.prepareReply("POST", - "https://api-staging.ultimaker.com/connect/v1/clusters/{}/print/{}" + CuraCloudAPIRoot + "/connect/v1/clusters/{}/print/{}" .format(cluster_id, job_id), 200, response) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py index fded79e15b..90a1b2fa96 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py @@ -7,6 +7,7 @@ from unittest.mock import patch, MagicMock from UM.Scene.SceneNode import SceneNode from UM.Signal import Signal from cura.CuraApplication import CuraApplication +from cura.CuraConstants import CuraCloudAPIRoot from cura.PrinterOutput.PrinterOutputModel import PrinterOutputModel from src.Cloud.CloudApiClient import CloudApiClient from src.Cloud.CloudOutputDevice import CloudOutputDevice @@ -23,10 +24,9 @@ class TestCloudOutputDevice(TestCase): HOST_NAME = "ultimakersystem-ccbdd30044ec" HOST_GUID = "e90ae0ac-1257-4403-91ee-a44c9b7e8050" - BASE_URL = "https://api-staging.ultimaker.com" - STATUS_URL = "{}/connect/v1/clusters/{}/status".format(BASE_URL, CLUSTER_ID) - PRINT_URL = "{}/connect/v1/clusters/{}/print/{}".format(BASE_URL, CLUSTER_ID, JOB_ID) - REQUEST_UPLOAD_URL = "{}/cura/v1/jobs/upload".format(BASE_URL) + STATUS_URL = "{}/connect/v1/clusters/{}/status".format(CuraCloudAPIRoot, CLUSTER_ID) + PRINT_URL = "{}/connect/v1/clusters/{}/print/{}".format(CuraCloudAPIRoot, CLUSTER_ID, JOB_ID) + REQUEST_UPLOAD_URL = "{}/cura/v1/jobs/upload".format(CuraCloudAPIRoot) def setUp(self): super().setUp() diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py index f62d92e9db..01b1b18ff1 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py @@ -4,6 +4,7 @@ from unittest import TestCase from unittest.mock import patch from cura.CuraApplication import CuraApplication +from cura.CuraConstants import CuraCloudAPIRoot from src.Cloud.CloudOutputDevice import CloudOutputDevice from src.Cloud.CloudOutputDeviceManager import CloudOutputDeviceManager from tests.Cloud.Fixtures import parseFixture, readFixture @@ -13,7 +14,7 @@ from .NetworkManagerMock import NetworkManagerMock class TestCloudOutputDeviceManager(TestCase): maxDiff = None - URL = "https://api-staging.ultimaker.com/connect/v1/clusters" + URL = CuraCloudAPIRoot + "/connect/v1/clusters" def setUp(self): super().setUp() From 3b367938de9600a94829bfaa8ac29e069cb49552 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 13:53:31 +0100 Subject: [PATCH 11/16] STAR-322: Removing TestSendMaterialJob temporarily --- .../tests/Cloud/TestCloudApiClient.py | 6 +- .../tests/TestSendMaterialJob.py | 190 ------------------ plugins/UM3NetworkPrinting/tests/__init__.py | 2 + run_mypy.py | 1 + 4 files changed, 5 insertions(+), 194 deletions(-) delete mode 100644 plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py create mode 100644 plugins/UM3NetworkPrinting/tests/__init__.py diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py index a09894deb0..49d2b1b34b 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py @@ -54,10 +54,8 @@ class TestCloudApiClient(TestCase): response = readFixture("getClusterStatusResponse") data = parseFixture("getClusterStatusResponse")["data"] - self.network.prepareReply("GET", - CuraCloudAPIRoot + "/connect/v1/clusters/R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC/status", - 200, response - ) + url = CuraCloudAPIRoot + "/connect/v1/clusters/R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC/status" + self.network.prepareReply("GET", url, 200, response) self.api.getClusterStatus("R0YcLJwar1ugh0ikEZsZs8NWKV6vJP_LdYsXgXqAcaNC", lambda s: result.append(s)) self.network.flushReplies() diff --git a/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py b/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py deleted file mode 100644 index 7db5ebdedf..0000000000 --- a/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright (c) 2018 Ultimaker B.V. -# Copyright (c) 2018 Ultimaker B.V. -# Cura is released under the terms of the LGPLv3 or higher. -import io -import json -from unittest import TestCase, mock -from unittest.mock import patch, call - -from PyQt5.QtCore import QByteArray - -from UM.MimeTypeDatabase import MimeType -from UM.Application import Application -from src.SendMaterialJob import SendMaterialJob - - -@patch("builtins.open", lambda _, __: io.StringIO("")) -@patch("UM.MimeTypeDatabase.MimeTypeDatabase.getMimeTypeForFile", - lambda _: MimeType(name = "application/x-ultimaker-material-profile", comment = "Ultimaker Material Profile", - suffixes = ["xml.fdm_material"])) -@patch("UM.Resources.Resources.getAllResourcesOfType", lambda _: ["/materials/generic_pla_white.xml.fdm_material"]) -@patch("plugins.UM3NetworkPrinting.src.ClusterUM3OutputDevice") -@patch("PyQt5.QtNetwork.QNetworkReply") -class TestSendMaterialJob(TestCase): - _LOCAL_MATERIAL_WHITE = {"type": "material", "status": "unknown", "id": "generic_pla_white", - "base_file": "generic_pla_white", "setting_version": "5", "name": "White PLA", - "brand": "Generic", "material": "PLA", "color_name": "White", - "GUID": "badb0ee7-87c8-4f3f-9398-938587b67dce", "version": "1", "color_code": "#ffffff", - "description": "Test PLA White", "adhesion_info": "Use glue.", "approximate_diameter": "3", - "properties": {"density": "1.00", "diameter": "2.85", "weight": "750"}, - "definition": "fdmprinter", "compatible": True} - - _LOCAL_MATERIAL_BLACK = {"type": "material", "status": "unknown", "id": "generic_pla_black", - "base_file": "generic_pla_black", "setting_version": "5", "name": "Yellow CPE", - "brand": "Ultimaker", "material": "CPE", "color_name": "Black", - "GUID": "5fbb362a-41f9-4818-bb43-15ea6df34aa4", "version": "1", "color_code": "#000000", - "description": "Test PLA Black", "adhesion_info": "Use glue.", "approximate_diameter": "3", - "properties": {"density": "1.01", "diameter": "2.85", "weight": "750"}, - "definition": "fdmprinter", "compatible": True} - - _REMOTE_MATERIAL_WHITE = { - "guid": "badb0ee7-87c8-4f3f-9398-938587b67dce", - "material": "PLA", - "brand": "Generic", - "version": 1, - "color": "White", - "density": 1.00 - } - - _REMOTE_MATERIAL_BLACK = { - "guid": "5fbb362a-41f9-4818-bb43-15ea6df34aa4", - "material": "PLA", - "brand": "Generic", - "version": 2, - "color": "Black", - "density": 1.00 - } - - def test_run(self, device_mock, reply_mock): - job = SendMaterialJob(device_mock) - job.run() - - # We expect the materials endpoint to be called when the job runs. - device_mock.get.assert_called_with("materials/", on_finished = job._onGetRemoteMaterials) - - def test__onGetRemoteMaterials_withFailedRequest(self, reply_mock, device_mock): - reply_mock.attribute.return_value = 404 - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - # We expect the device not to be called for any follow up. - self.assertEqual(0, device_mock.createFormPart.call_count) - - def test__onGetRemoteMaterials_withWrongEncoding(self, reply_mock, device_mock): - reply_mock.attribute.return_value = 200 - reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("cp500")) - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - # Given that the parsing fails we do no expect the device to be called for any follow up. - self.assertEqual(0, device_mock.createFormPart.call_count) - - def test__onGetRemoteMaterials_withBadJsonAnswer(self, reply_mock, device_mock): - reply_mock.attribute.return_value = 200 - reply_mock.readAll.return_value = QByteArray(b"Six sick hicks nick six slick bricks with picks and sticks.") - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - # Given that the parsing fails we do no expect the device to be called for any follow up. - self.assertEqual(0, device_mock.createFormPart.call_count) - - def test__onGetRemoteMaterials_withMissingGuidInRemoteMaterial(self, reply_mock, device_mock): - reply_mock.attribute.return_value = 200 - remote_material_without_guid = self._REMOTE_MATERIAL_WHITE.copy() - del remote_material_without_guid["guid"] - reply_mock.readAll.return_value = QByteArray(json.dumps([remote_material_without_guid]).encode("ascii")) - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - # Given that parsing fails we do not expect the device to be called for any follow up. - self.assertEqual(0, device_mock.createFormPart.call_count) - - @patch("cura.Settings.CuraContainerRegistry") - @patch("UM.Application") - def test__onGetRemoteMaterials_withInvalidVersionInLocalMaterial(self, application_mock, container_registry_mock, - reply_mock, device_mock): - reply_mock.attribute.return_value = 200 - reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) - - localMaterialWhiteWithInvalidVersion = self._LOCAL_MATERIAL_WHITE.copy() - localMaterialWhiteWithInvalidVersion["version"] = "one" - container_registry_mock.findContainersMetadata.return_value = [localMaterialWhiteWithInvalidVersion] - - application_mock.getContainerRegistry.return_value = container_registry_mock - - with mock.patch.object(Application, "getInstance", new = lambda: application_mock): - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - self.assertEqual(0, device_mock.createFormPart.call_count) - - @patch("cura.Settings.CuraContainerRegistry") - @patch("UM.Application") - def test__onGetRemoteMaterials_withNoUpdate(self, application_mock, container_registry_mock, reply_mock, - device_mock): - application_mock.getContainerRegistry.return_value = container_registry_mock - - device_mock.createFormPart.return_value = "_xXx_" - - container_registry_mock.findContainersMetadata.return_value = [self._LOCAL_MATERIAL_WHITE] - - reply_mock.attribute.return_value = 200 - reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) - - with mock.patch.object(Application, "getInstance", new = lambda: application_mock): - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - self.assertEqual(0, device_mock.createFormPart.call_count) - self.assertEqual(0, device_mock.postFormWithParts.call_count) - - @patch("cura.Settings.CuraContainerRegistry") - @patch("UM.Application") - def test__onGetRemoteMaterials_withUpdatedMaterial(self, application_mock, container_registry_mock, reply_mock, - device_mock): - application_mock.getContainerRegistry.return_value = container_registry_mock - - device_mock.createFormPart.return_value = "_xXx_" - - localMaterialWhiteWithHigherVersion = self._LOCAL_MATERIAL_WHITE.copy() - localMaterialWhiteWithHigherVersion["version"] = "2" - container_registry_mock.findContainersMetadata.return_value = [localMaterialWhiteWithHigherVersion] - - reply_mock.attribute.return_value = 200 - reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) - - with mock.patch.object(Application, "getInstance", new = lambda: application_mock): - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - self.assertEqual(1, device_mock.createFormPart.call_count) - self.assertEqual(1, device_mock.postFormWithParts.call_count) - self.assertEquals( - [call.createFormPart("name=\"file\"; filename=\"generic_pla_white.xml.fdm_material\"", ""), - call.postFormWithParts(target = "materials/", parts = ["_xXx_"], on_finished = job.sendingFinished)], - device_mock.method_calls) - - @patch("cura.Settings.CuraContainerRegistry") - @patch("UM.Application") - def test__onGetRemoteMaterials_withNewMaterial(self, application_mock, container_registry_mock, reply_mock, - device_mock): - application_mock.getContainerRegistry.return_value = container_registry_mock - - device_mock.createFormPart.return_value = "_xXx_" - - container_registry_mock.findContainersMetadata.return_value = [self._LOCAL_MATERIAL_WHITE, - self._LOCAL_MATERIAL_BLACK] - - reply_mock.attribute.return_value = 200 - reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_BLACK]).encode("ascii")) - - with mock.patch.object(Application, "getInstance", new = lambda: application_mock): - job = SendMaterialJob(device_mock) - job._onGetRemoteMaterials(reply_mock) - - self.assertEqual(1, device_mock.createFormPart.call_count) - self.assertEqual(1, device_mock.postFormWithParts.call_count) - self.assertEquals( - [call.createFormPart("name=\"file\"; filename=\"generic_pla_white.xml.fdm_material\"", ""), - call.postFormWithParts(target = "materials/", parts = ["_xXx_"], on_finished = job.sendingFinished)], - device_mock.method_calls) diff --git a/plugins/UM3NetworkPrinting/tests/__init__.py b/plugins/UM3NetworkPrinting/tests/__init__.py new file mode 100644 index 0000000000..f3f6970c54 --- /dev/null +++ b/plugins/UM3NetworkPrinting/tests/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2018 Ultimaker B.V. +# Cura is released under the terms of the LGPLv3 or higher. diff --git a/run_mypy.py b/run_mypy.py index 27f07cd281..2073f0e9a7 100644 --- a/run_mypy.py +++ b/run_mypy.py @@ -29,6 +29,7 @@ def where(exe_name: str, search_path: str = os.getenv("PATH")) -> str: def findModules(path): + return ["UM3NetworkPrinting"] result = [] for entry in os.scandir(path): if entry.is_dir() and os.path.exists(os.path.join(path, entry.name, "__init__.py")): From 75d7d493499b4594166c8a7adf9d4be719065ff4 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 14:41:28 +0100 Subject: [PATCH 12/16] STAR-322: Mocking the CuraApp --- .../src/Cloud/CloudApiClient.py | 10 +- .../tests/Cloud/TestCloudApiClient.py | 2 - .../tests/Cloud/TestCloudOutputDevice.py | 18 +- .../Cloud/TestCloudOutputDeviceManager.py | 51 +++-- .../tests/TestSendMaterialJob.py | 190 ++++++++++++++++++ plugins/UM3NetworkPrinting/tests/conftest.py | 40 ---- 6 files changed, 238 insertions(+), 73 deletions(-) create mode 100644 plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py delete mode 100644 plugins/UM3NetworkPrinting/tests/conftest.py diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index d58def4545..c6fb02753f 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -52,7 +52,7 @@ class CloudApiClient: def getClusters(self, on_finished: Callable[[List[CloudClusterResponse]], Any]) -> None: url = "{}/clusters".format(self.CLUSTER_API_ROOT) reply = self._manager.get(self._createEmptyRequest(url)) - self._addCallbacks(reply, on_finished, CloudClusterResponse) + self._addCallback(reply, on_finished, CloudClusterResponse) ## Retrieves the status of the given cluster. # \param cluster_id: The ID of the cluster. @@ -60,7 +60,7 @@ class CloudApiClient: def getClusterStatus(self, cluster_id: str, on_finished: Callable[[CloudClusterStatus], Any]) -> None: url = "{}/clusters/{}/status".format(self.CLUSTER_API_ROOT, cluster_id) reply = self._manager.get(self._createEmptyRequest(url)) - self._addCallbacks(reply, on_finished, CloudClusterStatus) + self._addCallback(reply, on_finished, CloudClusterStatus) ## Requests the cloud to register the upload of a print job mesh. # \param request: The request object. @@ -70,7 +70,7 @@ class CloudApiClient: url = "{}/jobs/upload".format(self.CURA_API_ROOT) body = json.dumps({"data": request.toDict()}) reply = self._manager.put(self._createEmptyRequest(url), body.encode()) - self._addCallbacks(reply, on_finished, CloudPrintJobResponse) + self._addCallback(reply, on_finished, CloudPrintJobResponse) ## Requests the cloud to register the upload of a print job mesh. # \param upload_response: The object received after requesting an upload with `self.requestUpload`. @@ -90,7 +90,7 @@ class CloudApiClient: def requestPrint(self, cluster_id: str, job_id: str, on_finished: Callable[[CloudPrintResponse], Any]) -> None: url = "{}/clusters/{}/print/{}".format(self.CLUSTER_API_ROOT, cluster_id, job_id) reply = self._manager.post(self._createEmptyRequest(url), b"") - self._addCallbacks(reply, on_finished, CloudPrintResponse) + self._addCallback(reply, on_finished, CloudPrintResponse) ## We override _createEmptyRequest in order to add the user credentials. # \param url: The URL to request @@ -147,7 +147,7 @@ class CloudApiClient: # \param on_finished: The callback in case the response is successful. # \param model: The type of the model to convert the response to. It may either be a single record or a list. # \return: A function that can be passed to the - def _addCallbacks(self, + def _addCallback(self, reply: QNetworkReply, on_finished: Union[Callable[[Model], Any], Callable[[List[Model]], Any]], model: Type[Model], diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py index 49d2b1b34b..0c0c8cffdf 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudApiClient.py @@ -6,7 +6,6 @@ from typing import List from unittest import TestCase from unittest.mock import patch, MagicMock -from cura.CuraApplication import CuraApplication from cura.CuraConstants import CuraCloudAPIRoot from src.Cloud.CloudApiClient import CloudApiClient from src.Cloud.Models.CloudClusterResponse import CloudClusterResponse @@ -29,7 +28,6 @@ class TestCloudApiClient(TestCase): self.account = MagicMock() self.account.isLoggedIn.return_value = True - self.app = CuraApplication.getInstance() self.network = NetworkManagerMock() with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): self.api = CloudApiClient(self.account, self._errorHandler) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py index 90a1b2fa96..34e04689c2 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDevice.py @@ -30,9 +30,13 @@ class TestCloudOutputDevice(TestCase): def setUp(self): super().setUp() - self.app = CuraApplication.getInstance() - self.backend = MagicMock(backendStateChange = Signal()) - self.app.setBackend(self.backend) + self.app = MagicMock() + + self.patches = [patch("UM.Qt.QtApplication.QtApplication.getInstance", return_value=self.app), + patch("UM.Application.Application.getInstance", return_value=self.app)] + for patched_method in self.patches: + patched_method.start() + self.cluster = CloudClusterResponse(self.CLUSTER_ID, self.HOST_GUID, self.HOST_NAME, is_online=True, status="active") @@ -41,6 +45,7 @@ class TestCloudOutputDevice(TestCase): self.onError = MagicMock() with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): self._api = CloudApiClient(self.account, self.onError) + self.device = CloudOutputDevice(self._api, self.cluster) self.cluster_status = parseFixture("getClusterStatusResponse") self.network.prepareReply("GET", self.STATUS_URL, 200, readFixture("getClusterStatusResponse")) @@ -48,6 +53,8 @@ class TestCloudOutputDevice(TestCase): def tearDown(self): super().tearDown() self.network.flushReplies() + for patched_method in self.patches: + patched_method.stop() def test_status(self): self.device._update() @@ -105,9 +112,8 @@ class TestCloudOutputDevice(TestCase): self.network.flushReplies() self.assertEqual([], self.device.printers) - @patch("cura.CuraApplication.CuraApplication.getGlobalContainerStack") - def test_print_to_cloud(self, global_container_stack_mock): - active_machine_mock = global_container_stack_mock.return_value + def test_print_to_cloud(self): + active_machine_mock = self.app.getGlobalContainerStack.return_value active_machine_mock.getMetaDataEntry.side_effect = {"file_formats": "application/gzip"}.get request_upload_response = parseFixture("putJobUploadResponse") diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py index 01b1b18ff1..96137a3edb 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/TestCloudOutputDeviceManager.py @@ -1,14 +1,14 @@ # Copyright (c) 2018 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. from unittest import TestCase -from unittest.mock import patch +from unittest.mock import patch, MagicMock -from cura.CuraApplication import CuraApplication +from UM.OutputDevice.OutputDeviceManager import OutputDeviceManager from cura.CuraConstants import CuraCloudAPIRoot from src.Cloud.CloudOutputDevice import CloudOutputDevice from src.Cloud.CloudOutputDeviceManager import CloudOutputDeviceManager from tests.Cloud.Fixtures import parseFixture, readFixture -from .NetworkManagerMock import NetworkManagerMock +from .NetworkManagerMock import NetworkManagerMock, FakeSignal class TestCloudOutputDeviceManager(TestCase): @@ -18,9 +18,19 @@ class TestCloudOutputDeviceManager(TestCase): def setUp(self): super().setUp() - self.app = CuraApplication.getInstance() + self.app = MagicMock() + self.device_manager = OutputDeviceManager() + self.app.getOutputDeviceManager.return_value = self.device_manager + + self.patches = [patch("UM.Qt.QtApplication.QtApplication.getInstance", return_value=self.app), + patch("UM.Application.Application.getInstance", return_value=self.app)] + for patched_method in self.patches: + patched_method.start() + self.network = NetworkManagerMock() - with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network): + self.timer = MagicMock(timeout = FakeSignal()) + with patch("src.Cloud.CloudApiClient.QNetworkAccessManager", return_value = self.network), \ + patch("src.Cloud.CloudOutputDeviceManager.QTimer", return_value = self.timer): self.manager = CloudOutputDeviceManager() self.clusters_response = parseFixture("getClusters") self.network.prepareReply("GET", self.URL, 200, readFixture("getClusters")) @@ -28,7 +38,11 @@ class TestCloudOutputDeviceManager(TestCase): def tearDown(self): try: self._beforeTearDown() + + self.network.flushReplies() self.manager.stop() + for patched_method in self.patches: + patched_method.stop() finally: super().tearDown() @@ -38,8 +52,7 @@ class TestCloudOutputDeviceManager(TestCase): # let the network send replies self.network.flushReplies() # get the created devices - device_manager = self.app.getOutputDeviceManager() - devices = device_manager.getOutputDevices() + devices = self.device_manager.getOutputDevices() # get the server data clusters = self.clusters_response.get("data", []) self.assertEqual([CloudOutputDevice] * len(clusters), [type(d) for d in devices]) @@ -48,13 +61,12 @@ class TestCloudOutputDeviceManager(TestCase): key=lambda device_dict: device_dict["host_version"])) for device in clusters: - device_manager.getOutputDevice(device["cluster_id"]).close() - device_manager.removeOutputDevice(device["cluster_id"]) + self.device_manager.getOutputDevice(device["cluster_id"]).close() + self.device_manager.removeOutputDevice(device["cluster_id"]) ## Runs the initial request to retrieve the clusters. def _loadData(self): self.manager.start() - self.manager._onLoginStateChanged(is_logged_in = True) self.network.flushReplies() def test_device_is_created(self): @@ -79,22 +91,20 @@ class TestCloudOutputDeviceManager(TestCase): self.manager._update_timer.timeout.emit() - @patch("cura.CuraApplication.CuraApplication.getGlobalContainerStack") - def test_device_connects_by_cluster_id(self, global_container_stack_mock): - active_machine_mock = global_container_stack_mock.return_value + def test_device_connects_by_cluster_id(self): + active_machine_mock = self.app.getGlobalContainerStack.return_value cluster1, cluster2 = self.clusters_response["data"] cluster_id = cluster1["cluster_id"] active_machine_mock.getMetaDataEntry.side_effect = {"um_cloud_cluster_id": cluster_id}.get self._loadData() - self.assertTrue(self.app.getOutputDeviceManager().getOutputDevice(cluster1["cluster_id"]).isConnected()) - self.assertFalse(self.app.getOutputDeviceManager().getOutputDevice(cluster2["cluster_id"]).isConnected()) + self.assertTrue(self.device_manager.getOutputDevice(cluster1["cluster_id"]).isConnected()) + self.assertFalse(self.device_manager.getOutputDevice(cluster2["cluster_id"]).isConnected()) self.assertEquals([], active_machine_mock.setMetaDataEntry.mock_calls) - @patch("cura.CuraApplication.CuraApplication.getGlobalContainerStack") - def test_device_connects_by_network_key(self, global_container_stack_mock): - active_machine_mock = global_container_stack_mock.return_value + def test_device_connects_by_network_key(self): + active_machine_mock = self.app.getGlobalContainerStack.return_value cluster1, cluster2 = self.clusters_response["data"] network_key = cluster2["host_name"] + ".ultimaker.local" @@ -102,8 +112,9 @@ class TestCloudOutputDeviceManager(TestCase): self._loadData() - self.assertFalse(self.app.getOutputDeviceManager().getOutputDevice(cluster1["cluster_id"]).isConnected()) - self.assertTrue(self.app.getOutputDeviceManager().getOutputDevice(cluster2["cluster_id"]).isConnected()) + self.assertEqual([False, True], + [self.device_manager.getOutputDevice(cluster["cluster_id"]).isConnected() + for cluster in (cluster1, cluster2)]) active_machine_mock.setMetaDataEntry.assert_called_with("um_cloud_cluster_id", cluster2["cluster_id"]) diff --git a/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py b/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py new file mode 100644 index 0000000000..b669eb192a --- /dev/null +++ b/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py @@ -0,0 +1,190 @@ +# Copyright (c) 2018 Ultimaker B.V. +# Copyright (c) 2018 Ultimaker B.V. +# Cura is released under the terms of the LGPLv3 or higher. +import io +import json +from unittest import TestCase, mock +from unittest.mock import patch, call + +from PyQt5.QtCore import QByteArray + +from UM.MimeTypeDatabase import MimeType +from UM.Application import Application +from plugins.UM3NetworkPrinting.src.SendMaterialJob import SendMaterialJob + + +@patch("builtins.open", lambda _, __: io.StringIO("")) +@patch("UM.MimeTypeDatabase.MimeTypeDatabase.getMimeTypeForFile", + lambda _: MimeType(name = "application/x-ultimaker-material-profile", comment = "Ultimaker Material Profile", + suffixes = ["xml.fdm_material"])) +@patch("UM.Resources.Resources.getAllResourcesOfType", lambda _: ["/materials/generic_pla_white.xml.fdm_material"]) +@patch("plugins.UM3NetworkPrinting.src.ClusterUM3OutputDevice") +@patch("PyQt5.QtNetwork.QNetworkReply") +class TestSendMaterialJob(TestCase): + _LOCAL_MATERIAL_WHITE = {"type": "material", "status": "unknown", "id": "generic_pla_white", + "base_file": "generic_pla_white", "setting_version": "5", "name": "White PLA", + "brand": "Generic", "material": "PLA", "color_name": "White", + "GUID": "badb0ee7-87c8-4f3f-9398-938587b67dce", "version": "1", "color_code": "#ffffff", + "description": "Test PLA White", "adhesion_info": "Use glue.", "approximate_diameter": "3", + "properties": {"density": "1.00", "diameter": "2.85", "weight": "750"}, + "definition": "fdmprinter", "compatible": True} + + _LOCAL_MATERIAL_BLACK = {"type": "material", "status": "unknown", "id": "generic_pla_black", + "base_file": "generic_pla_black", "setting_version": "5", "name": "Yellow CPE", + "brand": "Ultimaker", "material": "CPE", "color_name": "Black", + "GUID": "5fbb362a-41f9-4818-bb43-15ea6df34aa4", "version": "1", "color_code": "#000000", + "description": "Test PLA Black", "adhesion_info": "Use glue.", "approximate_diameter": "3", + "properties": {"density": "1.01", "diameter": "2.85", "weight": "750"}, + "definition": "fdmprinter", "compatible": True} + + _REMOTE_MATERIAL_WHITE = { + "guid": "badb0ee7-87c8-4f3f-9398-938587b67dce", + "material": "PLA", + "brand": "Generic", + "version": 1, + "color": "White", + "density": 1.00 + } + + _REMOTE_MATERIAL_BLACK = { + "guid": "5fbb362a-41f9-4818-bb43-15ea6df34aa4", + "material": "PLA", + "brand": "Generic", + "version": 2, + "color": "Black", + "density": 1.00 + } + + def test_run(self, device_mock, reply_mock): + job = SendMaterialJob(device_mock) + job.run() + + # We expect the materials endpoint to be called when the job runs. + device_mock.get.assert_called_with("materials/", on_finished = job._onGetRemoteMaterials) + + def test__onGetRemoteMaterials_withFailedRequest(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 404 + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # We expect the device not to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + def test__onGetRemoteMaterials_withWrongEncoding(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("cp500")) + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # Given that the parsing fails we do no expect the device to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + def test__onGetRemoteMaterials_withBadJsonAnswer(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(b"Six sick hicks nick six slick bricks with picks and sticks.") + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # Given that the parsing fails we do no expect the device to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + def test__onGetRemoteMaterials_withMissingGuidInRemoteMaterial(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + remote_material_without_guid = self._REMOTE_MATERIAL_WHITE.copy() + del remote_material_without_guid["guid"] + reply_mock.readAll.return_value = QByteArray(json.dumps([remote_material_without_guid]).encode("ascii")) + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # Given that parsing fails we do not expect the device to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withInvalidVersionInLocalMaterial(self, application_mock, container_registry_mock, + reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) + + localMaterialWhiteWithInvalidVersion = self._LOCAL_MATERIAL_WHITE.copy() + localMaterialWhiteWithInvalidVersion["version"] = "one" + container_registry_mock.findContainersMetadata.return_value = [localMaterialWhiteWithInvalidVersion] + + application_mock.getContainerRegistry.return_value = container_registry_mock + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(0, device_mock.createFormPart.call_count) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withNoUpdate(self, application_mock, container_registry_mock, reply_mock, + device_mock): + application_mock.getContainerRegistry.return_value = container_registry_mock + + device_mock.createFormPart.return_value = "_xXx_" + + container_registry_mock.findContainersMetadata.return_value = [self._LOCAL_MATERIAL_WHITE] + + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(0, device_mock.createFormPart.call_count) + self.assertEqual(0, device_mock.postFormWithParts.call_count) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withUpdatedMaterial(self, application_mock, container_registry_mock, reply_mock, + device_mock): + application_mock.getContainerRegistry.return_value = container_registry_mock + + device_mock.createFormPart.return_value = "_xXx_" + + localMaterialWhiteWithHigherVersion = self._LOCAL_MATERIAL_WHITE.copy() + localMaterialWhiteWithHigherVersion["version"] = "2" + container_registry_mock.findContainersMetadata.return_value = [localMaterialWhiteWithHigherVersion] + + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(1, device_mock.createFormPart.call_count) + self.assertEqual(1, device_mock.postFormWithParts.call_count) + self.assertEquals( + [call.createFormPart("name=\"file\"; filename=\"generic_pla_white.xml.fdm_material\"", ""), + call.postFormWithParts(target = "materials/", parts = ["_xXx_"], on_finished = job.sendingFinished)], + device_mock.method_calls) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withNewMaterial(self, application_mock, container_registry_mock, reply_mock, + device_mock): + application_mock.getContainerRegistry.return_value = container_registry_mock + + device_mock.createFormPart.return_value = "_xXx_" + + container_registry_mock.findContainersMetadata.return_value = [self._LOCAL_MATERIAL_WHITE, + self._LOCAL_MATERIAL_BLACK] + + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_BLACK]).encode("ascii")) + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(1, device_mock.createFormPart.call_count) + self.assertEqual(1, device_mock.postFormWithParts.call_count) + self.assertEquals( + [call.createFormPart("name=\"file\"; filename=\"generic_pla_white.xml.fdm_material\"", ""), + call.postFormWithParts(target = "materials/", parts = ["_xXx_"], on_finished = job.sendingFinished)], + device_mock.method_calls) diff --git a/plugins/UM3NetworkPrinting/tests/conftest.py b/plugins/UM3NetworkPrinting/tests/conftest.py deleted file mode 100644 index ce49bd3cb7..0000000000 --- a/plugins/UM3NetworkPrinting/tests/conftest.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2018 Ultimaker B.V. -# Cura is released under the terms of the LGPLv3 or higher. - -import pytest -from UM.Signal import Signal - -from cura.CuraApplication import CuraApplication -from cura.Machines.MaterialManager import MaterialManager - - -# This mock application must extend from Application and not QtApplication otherwise some QObjects are created and -# a segfault is raised. -class FixtureApplication(CuraApplication): - def __init__(self): - super().__init__() - super().initialize() - Signal._signalQueue = self - - self.getPreferences().addPreference("cura/favorite_materials", "") - - self._material_manager = MaterialManager(self._container_registry, parent = self) - self._material_manager.initialize() - - def functionEvent(self, event): - event.call() - - def parseCommandLine(self): - pass - - def processEvents(self): - pass - - -@pytest.fixture(autouse=True) -def application(): - # Since we need to use it more that once, we create the application the first time and use its instance the second - application = FixtureApplication.getInstance() - if application is None: - application = FixtureApplication() - return application From 42ae9faeb4df483be003b6f45c7f91605402d54f Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 14:42:16 +0100 Subject: [PATCH 13/16] STAR-322: Reverting change to run mypy --- run_mypy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run_mypy.py b/run_mypy.py index 2073f0e9a7..27f07cd281 100644 --- a/run_mypy.py +++ b/run_mypy.py @@ -29,7 +29,6 @@ def where(exe_name: str, search_path: str = os.getenv("PATH")) -> str: def findModules(path): - return ["UM3NetworkPrinting"] result = [] for entry in os.scandir(path): if entry.is_dir() and os.path.exists(os.path.join(path, entry.name, "__init__.py")): From da974d98686356eee9a39335334841d02fe68555 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 14:51:57 +0100 Subject: [PATCH 14/16] STAR-322: Improving documentation --- .../src/Cloud/CloudApiClient.py | 32 +++++++++++-------- .../tests/Cloud/NetworkManagerMock.py | 4 +-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py index c6fb02753f..29a9f48c3f 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudApiClient.py @@ -72,12 +72,12 @@ class CloudApiClient: reply = self._manager.put(self._createEmptyRequest(url), body.encode()) self._addCallback(reply, on_finished, CloudPrintJobResponse) - ## Requests the cloud to register the upload of a print job mesh. - # \param upload_response: The object received after requesting an upload with `self.requestUpload`. + ## Uploads a print job mesh to the cloud. + # \param print_job: The object received after requesting an upload with `self.requestUpload`. # \param mesh: The mesh data to be uploaded. - # \param on_finished: The function to be called after the result is parsed. It receives the print job ID. + # \param on_finished: The function to be called after the upload is successful. # \param on_progress: A function to be called during upload progress. It receives a percentage (0-100). - # \param on_error: A function to be called if the upload fails. It receives a dict with the error. + # \param on_error: A function to be called if the upload fails. def uploadMesh(self, print_job: CloudPrintJobResponse, mesh: bytes, on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any]): self._upload = MeshUploader(self._manager, print_job, mesh, on_finished, on_progress, on_error) @@ -134,24 +134,28 @@ class CloudApiClient: data = response["data"] if isinstance(data, list): results = [model_class(**c) for c in data] # type: List[CloudApiClient.Model] - cast(Callable[[List[CloudApiClient.Model]], Any], on_finished)(results) + on_finished_list = cast(Callable[[List[CloudApiClient.Model]], Any], on_finished) + on_finished_list(results) else: result = model_class(**data) # type: CloudApiClient.Model - cast(Callable[[CloudApiClient.Model], Any], on_finished)(result) + on_finished_item = cast(Callable[[CloudApiClient.Model], Any], on_finished) + on_finished_item(result) elif "errors" in response: self._on_error([CloudErrorObject(**error) for error in response["errors"]]) else: Logger.log("e", "Cannot find data or errors in the cloud response: %s", response) - ## Wraps a callback function so that it includes the parsing of the response into the correct model. - # \param on_finished: The callback in case the response is successful. - # \param model: The type of the model to convert the response to. It may either be a single record or a list. - # \return: A function that can be passed to the + ## Creates a callback function so that it includes the parsing of the response into the correct model. + # The callback is added to the 'finished' signal of the reply. + # \param reply: The reply that should be listened to. + # \param on_finished: The callback in case the response is successful. Depending on the endpoint it will be either + # a list or a single item. + # \param model: The type of the model to convert the response to. def _addCallback(self, - reply: QNetworkReply, - on_finished: Union[Callable[[Model], Any], Callable[[List[Model]], Any]], - model: Type[Model], - ) -> None: + reply: QNetworkReply, + on_finished: Union[Callable[[Model], Any], Callable[[List[Model]], Any]], + model: Type[Model], + ) -> None: def parse() -> None: status_code, response = self._parseReply(reply) self._anti_gc_callbacks.remove(parse) diff --git a/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py b/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py index 59b79fdfa6..5b5d89ca54 100644 --- a/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py +++ b/plugins/UM3NetworkPrinting/tests/Cloud/NetworkManagerMock.py @@ -1,7 +1,7 @@ # Copyright (c) 2018 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. import json -from typing import Dict, Tuple, Union, Optional +from typing import Dict, Tuple, Union, Optional, Any from unittest.mock import MagicMock from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest @@ -53,7 +53,7 @@ class NetworkManagerMock: # Since the methods are very simple and the same it didn't make sense to repeat the code. # \param method: The method being called. # \return The mocked function, if the method name is known. Defaults to the standard getattr function. - def __getattr__(self, method: str) -> any: + def __getattr__(self, method: str) -> Any: ## This mock implementation will simply return the reply from the prepared ones. # it raises a KeyError if requests are done without being prepared. def doRequest(request: QNetworkRequest, body: Optional[bytes] = None, *_): From d28ac5e1205f8b5d3083807c2975925c78052e9b Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 14:57:57 +0100 Subject: [PATCH 15/16] STAR-322: Improving documentation --- .../src/Cloud/CloudOutputDevice.py | 19 +++++++++---------- .../src/Cloud/CloudOutputDeviceManager.py | 2 ++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py index f29e29c40b..fed8cb040a 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDevice.py @@ -83,7 +83,7 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): ## Creates a new cloud output device # \param api_client: The client that will run the API calls - # \param device_id: The ID of the device (i.e. the cluster_id for the cloud API) + # \param cluster: The device response received from the cloud API. # \param parent: The optional parent of this output device. def __init__(self, api_client: CloudApiClient, cluster: CloudClusterResponse, parent: QObject = None) -> None: super().__init__(device_id = cluster.cluster_id, address = "", properties = {}, parent = parent) @@ -118,30 +118,33 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # A set of the user's job IDs that have finished self._finished_jobs = set() # type: Set[str] - # Reference to the uploaded print job + # Reference to the uploaded print job / mesh self._mesh = None # type: Optional[bytes] self._uploaded_print_job = None # type: Optional[CloudPrintJobResponse] + ## Connects this device. def connect(self) -> None: super().connect() Logger.log("i", "Connected to cluster %s", self.key) CuraApplication.getInstance().getBackend().backendStateChange.connect(self._onBackendStateChange) + ## Disconnects the device def disconnect(self) -> None: super().disconnect() Logger.log("i", "Disconnected to cluster %s", self.key) CuraApplication.getInstance().getBackend().backendStateChange.disconnect(self._onBackendStateChange) + ## Resets the print job that was uploaded to force a new upload, runs whenever the user re-slices. def _onBackendStateChange(self, _: BackendState) -> None: self._mesh = None self._uploaded_print_job = None - ## Gets the host name of this device + ## Gets the cluster response from which this device was created. @property def clusterData(self) -> CloudClusterResponse: return self._cluster - ## Updates the host name of the output device + ## Updates the cluster data from the cloud. @clusterData.setter def clusterData(self, value: CloudClusterResponse) -> None: self._cluster = value @@ -166,11 +169,8 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): # Show an error message if we're already sending a job. if self._progress.visible: - Message( - text = T.BLOCKED_UPLOADING, - title = T.ERROR, - lifetime = 10, - ).show() + message = Message(text = T.BLOCKED_UPLOADING, title = T.ERROR, lifetime = 10) + message.show() return if self._uploaded_print_job: @@ -312,7 +312,6 @@ class CloudOutputDevice(NetworkedPrinterOutputDevice): model.updateAssignedPrinter(printer) ## Uploads the mesh when the print job was registered with the cloud API. - # \param mesh: The bytes to upload. # \param job_response: The response received from the cloud API. def _onPrintJobCreated(self, job_response: CloudPrintJobResponse) -> None: self._progress.show() diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py index 29c60fd14a..07051f15fd 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudOutputDeviceManager.py @@ -139,6 +139,7 @@ class CloudOutputDeviceManager: ) message.show() + ## Starts running the cloud output device manager, thus periodically requesting cloud data. def start(self): if self._running: return @@ -149,6 +150,7 @@ class CloudOutputDeviceManager: self._update_timer.timeout.connect(self._getRemoteClusters) self._onLoginStateChanged(is_logged_in = self._account.isLoggedIn) + ## Stops running the cloud output device manager. def stop(self): if not self._running: return From c5f438819a60c547b9d416696a6fc4665b539aa3 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 17 Dec 2018 15:11:01 +0100 Subject: [PATCH 16/16] STAR-322: Improving documentation --- .../src/Cloud/CloudProgressMessage.py | 5 ++ .../src/Cloud/MeshUploader.py | 54 ++++++++++++++----- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py b/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py index e3e0cefc0c..aefe59cc85 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/CloudProgressMessage.py @@ -12,6 +12,7 @@ class T: SENDING_DATA_TITLE = _I18N_CATALOG.i18nc("@info:status", "Sending data to remote cluster") +## Class responsible for showing a progress message while a mesh is being uploaded to the cloud. class CloudProgressMessage(Message): def __init__(self): super().__init__( @@ -23,15 +24,19 @@ class CloudProgressMessage(Message): use_inactivity_timer = False ) + ## Shows the progress message. def show(self): self.setProgress(0) super().show() + ## Updates the percentage of the uploaded. + # \param percentage: The percentage amount (0-100). def update(self, percentage: int) -> None: if not self._visible: super().show() self.setProgress(percentage) + ## Returns a boolean indicating whether the message is currently visible. @property def visible(self) -> bool: return self._visible diff --git a/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py b/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py index f0360b83e3..9d9662a82a 100644 --- a/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py +++ b/plugins/UM3NetworkPrinting/src/Cloud/MeshUploader.py @@ -9,16 +9,25 @@ from UM.Logger import Logger from src.Cloud.Models.CloudPrintJobResponse import CloudPrintJobResponse +## Class responsible for uploading meshes to the cloud in separate requests. class MeshUploader: + + # The maximum amount of times to retry if the server returns one of the RETRY_HTTP_CODES MAX_RETRIES = 10 - BYTES_PER_REQUEST = 256 * 1024 + + # The HTTP codes that should trigger a retry. RETRY_HTTP_CODES = {500, 502, 503, 504} - ## Creates a resumable upload - # \param url: The URL to which we shall upload. - # \param content_length: The total content length of the file, in bytes. - # \param http_method: The HTTP method to be used, e.g. "POST" or "PUT". - # \param timeout: The timeout for each chunk upload. Important: If None, no timeout is applied at all. + # The amount of bytes to send per request + BYTES_PER_REQUEST = 256 * 1024 + + ## Creates a mesh upload object. + # \param manager: The network access manager that will handle the HTTP requests. + # \param print_job: The print job response that was returned by the cloud after registering the upload. + # \param data: The mesh bytes to be uploaded. + # \param on_finished: The method to be called when done. + # \param on_progress: The method to be called when the progress changes (receives a percentage 0-100). + # \param on_error: The method to be called when an error occurs. def __init__(self, manager: QNetworkAccessManager, print_job: CloudPrintJobResponse, data: bytes, on_finished: Callable[[], Any], on_progress: Callable[[int], Any], on_error: Callable[[], Any] ) -> None: @@ -35,13 +44,12 @@ class MeshUploader: self._finished = False self._reply = None # type: Optional[QNetworkReply] + ## Returns the print job for which this object was created. @property def printJob(self): return self._print_job - ## We override _createRequest in order to add the user credentials. - # \param url: The URL to request - # \param content_type: The type of the body contents. + ## Creates a network request to the print job upload URL, adding the needed content range header. def _createRequest(self) -> QNetworkRequest: request = QNetworkRequest(QUrl(self._print_job.upload_url)) request.setHeader(QNetworkRequest.ContentTypeHeader, self._print_job.content_type) @@ -53,21 +61,27 @@ class MeshUploader: return request + ## Determines the bytes that should be uploaded next. + # \return: A tuple with the first and the last byte to upload. def _chunkRange(self) -> Tuple[int, int]: last_byte = min(len(self._data), self._sent_bytes + self.BYTES_PER_REQUEST) return self._sent_bytes, last_byte + ## Starts uploading the mesh. def start(self) -> None: if self._finished: + # reset state. self._sent_bytes = 0 self._retries = 0 self._finished = False self._uploadChunk() + ## Stops uploading the mesh, marking it as finished. def stop(self): Logger.log("i", "Stopped uploading") self._finished = True + ## Uploads a chunk of the mesh to the cloud. def _uploadChunk(self) -> None: if self._finished: raise ValueError("The upload is already finished") @@ -75,16 +89,22 @@ class MeshUploader: first_byte, last_byte = self._chunkRange() request = self._createRequest() + # now send the reply and subscribe to the results self._reply = self._manager.put(request, self._data[first_byte:last_byte]) self._reply.finished.connect(self._finishedCallback) self._reply.uploadProgress.connect(self._progressCallback) self._reply.error.connect(self._errorCallback) + ## Handles an update to the upload progress + # \param bytes_sent: The amount of bytes sent in the current request. + # \param bytes_total: The amount of bytes to send in the current request. def _progressCallback(self, bytes_sent: int, bytes_total: int) -> None: Logger.log("i", "Progress callback %s / %s", bytes_sent, bytes_total) if bytes_total: - self._on_progress(int((self._sent_bytes + bytes_sent) / len(self._data) * 100)) + total_sent = self._sent_bytes + bytes_sent + self._on_progress(int(total_sent / len(self._data) * 100)) + ## Handles an error uploading. def _errorCallback(self) -> None: reply = cast(QNetworkReply, self._reply) body = bytes(reply.readAll()).decode() @@ -92,27 +112,33 @@ class MeshUploader: self.stop() self._on_error() + ## Checks whether a chunk of data was uploaded successfully, starting the next chunk if needed. def _finishedCallback(self) -> None: reply = cast(QNetworkReply, self._reply) Logger.log("i", "Finished callback %s %s", reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.url().toString()) - status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) + status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) # type: int + # check if we should retry the last chunk if self._retries < self.MAX_RETRIES and status_code in self.RETRY_HTTP_CODES: self._retries += 1 Logger.log("i", "Retrying %s/%s request %s", self._retries, self.MAX_RETRIES, reply.url().toString()) self._uploadChunk() return + # Http codes that are not to be retried are assumed to be errors. if status_code > 308: self._errorCallback() return - body = bytes(reply.readAll()).decode() - Logger.log("w", "status_code: %s, Headers: %s, body: %s", status_code, - [bytes(header).decode() for header in reply.rawHeaderList()], body) + Logger.log("d", "status_code: %s, Headers: %s, body: %s", status_code, + [bytes(header).decode() for header in reply.rawHeaderList()], bytes(reply.readAll()).decode()) + self._chunkUploaded() + ## Handles a chunk of data being uploaded, starting the next chunk if needed. + def _chunkUploaded(self) -> None: + # We got a successful response. Let's start the next chunk or report the upload is finished. first_byte, last_byte = self._chunkRange() self._sent_bytes += last_byte - first_byte if self._sent_bytes >= len(self._data):