diff --git a/cura/PrinterOutput/NetworkedPrinterOutputDevice.py b/cura/PrinterOutput/NetworkedPrinterOutputDevice.py index 35d2ce014a..9a3be936a2 100644 --- a/cura/PrinterOutput/NetworkedPrinterOutputDevice.py +++ b/cura/PrinterOutput/NetworkedPrinterOutputDevice.py @@ -147,6 +147,9 @@ class NetworkedPrinterOutputDevice(PrinterOutputDevice): request.setHeader(QNetworkRequest.UserAgentHeader, self._user_agent) return request + def createFormPart(self, content_header: str, data: bytes, content_type: Optional[str] = None) -> QHttpPart: + return self._createFormPart(content_header, data, content_type) + def _createFormPart(self, content_header: str, data: bytes, content_type: Optional[str] = None) -> QHttpPart: part = QHttpPart() diff --git a/plugins/UM3NetworkPrinting/src/Models.py b/plugins/UM3NetworkPrinting/src/Models.py new file mode 100644 index 0000000000..2bcac70766 --- /dev/null +++ b/plugins/UM3NetworkPrinting/src/Models.py @@ -0,0 +1,43 @@ +# Copyright (c) 2018 Ultimaker B.V. +# Cura is released under the terms of the LGPLv3 or higher. + + +## Base model that maps kwargs to instance attributes. +class BaseModel: + def __init__(self, **kwargs) -> None: + self.__dict__.update(kwargs) + self.validate() + + def validate(self) -> None: + pass + + +## Class representing a material that was fetched from the cluster API. +class ClusterMaterial(BaseModel): + def __init__(self, guid: str, version: int, **kwargs) -> None: + self.guid = guid # type: str + self.version = version # type: int + super().__init__(**kwargs) + + def validate(self) -> None: + if not self.guid: + raise ValueError("guid is required on ClusterMaterial") + if not self.version: + raise ValueError("version is required on ClusterMaterial") + + +## Class representing a local material that was fetched from the container registry. +class LocalMaterial(BaseModel): + def __init__(self, GUID: str, id: str, version: int, **kwargs) -> None: + self.GUID = GUID # type: str + self.id = id # type: str + self.version = version # type: int + super().__init__(**kwargs) + + def validate(self) -> None: + if not self.GUID: + raise ValueError("guid is required on LocalMaterial") + if not self.version: + raise ValueError("version is required on LocalMaterial") + if not self.id: + raise ValueError("id is required on LocalMaterial") diff --git a/plugins/UM3NetworkPrinting/src/SendMaterialJob.py b/plugins/UM3NetworkPrinting/src/SendMaterialJob.py index 8491e79c29..f536fad49a 100644 --- a/plugins/UM3NetworkPrinting/src/SendMaterialJob.py +++ b/plugins/UM3NetworkPrinting/src/SendMaterialJob.py @@ -1,99 +1,197 @@ # Copyright (c) 2018 Ultimaker B.V. # Cura is released under the terms of the LGPLv3 or higher. +import json +import os +import urllib.parse +from typing import Dict, TYPE_CHECKING, Set -import json #To understand the list of materials from the printer reply. -import os #To walk over material files. -import os.path #To filter on material files. -from PyQt5.QtNetwork import QNetworkReply, QNetworkRequest #To listen to the reply from the printer. -from typing import Any, Dict, Set, TYPE_CHECKING -import urllib.parse #For getting material IDs from their file names. +from PyQt5.QtNetwork import QNetworkReply, QNetworkRequest -from UM.Job import Job #The interface we're implementing. +from UM.Application import Application +from UM.Job import Job from UM.Logger import Logger -from UM.MimeTypeDatabase import MimeTypeDatabase #To strip the extensions of the material profile files. +from UM.MimeTypeDatabase import MimeTypeDatabase from UM.Resources import Resources -from UM.Settings.ContainerRegistry import ContainerRegistry #To find the GUIDs of materials. - -from cura.CuraApplication import CuraApplication #For the resource types. +from cura.CuraApplication import CuraApplication +# Absolute imports don't work in plugins +from .Models import ClusterMaterial, LocalMaterial if TYPE_CHECKING: from .ClusterUM3OutputDevice import ClusterUM3OutputDevice + ## Asynchronous job to send material profiles to the printer. # # This way it won't freeze up the interface while sending those materials. class SendMaterialJob(Job): + def __init__(self, device: "ClusterUM3OutputDevice") -> None: super().__init__() - self.device = device #type: ClusterUM3OutputDevice + self.device = device # type: ClusterUM3OutputDevice + ## Send the request to the printer and register a callback def run(self) -> None: - self.device.get("materials/", on_finished = self.sendMissingMaterials) + self.device.get("materials/", on_finished = self._onGetRemoteMaterials) - def sendMissingMaterials(self, reply: QNetworkReply) -> None: - if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) != 200: #Got an error from the HTTP request. - Logger.log("e", "Couldn't request current material storage on printer. Not syncing materials.") + ## Process the materials reply from the printer. + # + # \param reply The reply from the printer, a json file. + def _onGetRemoteMaterials(self, reply: QNetworkReply) -> None: + + # Got an error from the HTTP request. If we did not receive a 200 something happened. + if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) != 200: + Logger.log("e", "Error fetching materials from printer: %s", reply.errorString()) return - remote_materials_list = reply.readAll().data().decode("utf-8") - try: - remote_materials_list = json.loads(remote_materials_list) - except json.JSONDecodeError: - Logger.log("e", "Request material storage on printer: I didn't understand the printer's answer.") - return - try: - remote_materials_by_guid = {material["guid"]: material for material in remote_materials_list} #Index by GUID. - except KeyError: - Logger.log("e", "Request material storage on printer: Printer's answer was missing GUIDs.") + # Collect materials from the printer's reply and send the missing ones if needed. + remote_materials_by_guid = self._parseReply(reply) + if remote_materials_by_guid: + self._sendMissingMaterials(remote_materials_by_guid) + + ## Determine which materials should be updated and send them to the printer. + # + # \param remote_materials_by_guid The remote materials by GUID. + def _sendMissingMaterials(self, remote_materials_by_guid: Dict[str, ClusterMaterial]) -> None: + + # Collect local materials + local_materials_by_guid = self._getLocalMaterials() + if len(local_materials_by_guid) == 0: + Logger.log("d", "There are no local materials to synchronize with the printer.") return - container_registry = ContainerRegistry.getInstance() - local_materials_list = filter(lambda material: ("GUID" in material and "version" in material and "id" in material), container_registry.findContainersMetadata(type = "material")) - local_materials_by_guid = {material["GUID"]: material for material in local_materials_list if material["id"] == material["base_file"]} - for material in local_materials_list: #For each GUID get the material with the highest version number. - try: - if int(material["version"]) > local_materials_by_guid[material["GUID"]]["version"]: - local_materials_by_guid[material["GUID"]] = material - except ValueError: - Logger.log("e", "Material {material_id} has invalid version number {number}.".format(material_id = material["id"], number = material["version"])) - continue + # Find out what materials are new or updated and must be sent to the printer + material_ids_to_send = self._determineMaterialsToSend(local_materials_by_guid, remote_materials_by_guid) + if len(material_ids_to_send) == 0: + Logger.log("d", "There are no remote materials to update.") + return - materials_to_send = set() #type: Set[Dict[str, Any]] - for guid, material in local_materials_by_guid.items(): - if guid not in remote_materials_by_guid: - materials_to_send.add(material["id"]) - continue - try: - if int(material["version"]) > remote_materials_by_guid[guid]["version"]: - materials_to_send.add(material["id"]) - continue - except KeyError: - Logger.log("e", "Current material storage on printer was an invalid reply (missing version).") - return + # Send materials to the printer + self._sendMaterials(material_ids_to_send) - for file_path in Resources.getAllResourcesOfType(CuraApplication.ResourceTypes.MaterialInstanceContainer): + ## From the local and remote materials, determine which ones should be synchronized. + # + # Makes a Set of id's containing only the id's of the materials that are not on the printer yet or the ones that + # are newer in Cura. + # + # \param local_materials The local materials by GUID. + # \param remote_materials The remote materials by GUID. + @staticmethod + def _determineMaterialsToSend(local_materials: Dict[str, LocalMaterial], + remote_materials: Dict[str, ClusterMaterial]) -> Set[str]: + return { + material.id + for guid, material in local_materials.items() + if guid not in remote_materials or material.version > remote_materials[guid].version + } + + ## Send the materials to the printer. + # + # The given materials will be loaded from disk en sent to to printer. + # The given id's will be matched with filenames of the locally stored materials. + # + # \param materials_to_send A set with id's of materials that must be sent. + def _sendMaterials(self, materials_to_send: Set[str]) -> None: + file_paths = Resources.getAllResourcesOfType(CuraApplication.ResourceTypes.MaterialInstanceContainer) + + # Find all local material files and send them if needed. + for file_path in file_paths: try: mime_type = MimeTypeDatabase.getMimeTypeForFile(file_path) except MimeTypeDatabase.MimeTypeNotFoundError: - continue #Not the sort of file we'd like to send then. - _, file_name = os.path.split(file_path) - material_id = urllib.parse.unquote_plus(mime_type.stripExtension(file_name)) - if material_id not in materials_to_send: continue - parts = [] - with open(file_path, "rb") as f: - parts.append(self.device._createFormPart("name=\"file\"; filename=\"{file_name}\"".format(file_name = file_name), f.read())) - signature_file_path = file_path + ".sig" - if os.path.exists(signature_file_path): - _, signature_file_name = os.path.split(signature_file_path) - with open(signature_file_path, "rb") as f: - parts.append(self.device._createFormPart("name=\"signature_file\"; filename=\"{file_name}\"".format(file_name = signature_file_name), f.read())) + file_name = os.path.basename(file_path) + material_id = urllib.parse.unquote_plus(mime_type.stripExtension(file_name)) + if material_id not in materials_to_send: + # If the material does not have to be sent we skip it. + continue - Logger.log("d", "Syncing material {material_id} with cluster.".format(material_id = material_id)) - self.device.postFormWithParts(target = "materials/", parts = parts, on_finished = self.sendingFinished) + self._sendMaterialFile(file_path, file_name, material_id) - def sendingFinished(self, reply: QNetworkReply): + ## Send a single material file to the printer. + # + # Also add the material signature file if that is available. + # + # \param file_path The path of the material file. + # \param file_name The name of the material file. + # \param material_id The ID of the material in the file. + def _sendMaterialFile(self, file_path: str, file_name: str, material_id: str) -> None: + + parts = [] + + # Add the material file. + with open(file_path, "rb") as f: + parts.append(self.device.createFormPart("name=\"file\"; filename=\"{file_name}\"" + .format(file_name = file_name), f.read())) + + # Add the material signature file if needed. + signature_file_path = "{}.sig".format(file_path) + if os.path.exists(signature_file_path): + signature_file_name = os.path.basename(signature_file_path) + with open(signature_file_path, "rb") as f: + parts.append(self.device.createFormPart("name=\"signature_file\"; filename=\"{file_name}\"" + .format(file_name = signature_file_name), f.read())) + + Logger.log("d", "Syncing material {material_id} with cluster.".format(material_id = material_id)) + self.device.postFormWithParts(target = "materials/", parts = parts, on_finished = self.sendingFinished) + + ## Check a reply from an upload to the printer and log an error when the call failed + @staticmethod + def sendingFinished(reply: QNetworkReply) -> None: if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) != 200: - Logger.log("e", "Received error code from printer when syncing material: {code}".format(code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute))) - Logger.log("e", reply.readAll().data().decode("utf-8")) \ No newline at end of file + Logger.log("e", "Received error code from printer when syncing material: {code}, {text}".format( + code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), + text = reply.errorString() + )) + + ## Parse the reply from the printer + # + # Parses the reply to a "/materials" request to the printer + # + # \return a dictionary of ClusterMaterial objects by GUID + # \throw KeyError Raised when on of the materials does not include a valid guid + @classmethod + def _parseReply(cls, reply: QNetworkReply) -> Dict[str, ClusterMaterial]: + try: + remote_materials = json.loads(reply.readAll().data().decode("utf-8")) + return {material["guid"]: ClusterMaterial(**material) for material in remote_materials} + except UnicodeDecodeError: + Logger.log("e", "Request material storage on printer: I didn't understand the printer's answer.") + except json.JSONDecodeError: + Logger.log("e", "Request material storage on printer: I didn't understand the printer's answer.") + except ValueError: + Logger.log("e", "Request material storage on printer: Printer's answer had an incorrect value.") + except TypeError: + Logger.log("e", "Request material storage on printer: Printer's answer was missing a required value.") + + ## Retrieves a list of local materials + # + # Only the new newest version of the local materials is returned + # + # \return a dictionary of LocalMaterial objects by GUID + def _getLocalMaterials(self) -> Dict[str, LocalMaterial]: + result = {} # type: Dict[str, LocalMaterial] + container_registry = Application.getInstance().getContainerRegistry() + material_containers = container_registry.findContainersMetadata(type = "material") + + # Find the latest version of all material containers in the registry. + for material in material_containers: + try: + # material version must be an int + material["version"] = int(material["version"]) + + # Create a new local material + local_material = LocalMaterial(**material) + + if local_material.GUID not in result or \ + local_material.version > result.get(local_material.GUID).version: + result[local_material.GUID] = local_material + + except KeyError: + Logger.logException("w", "Local material {} has missing values.".format(material["id"])) + except ValueError: + Logger.logException("w", "Local material {} has invalid values.".format(material["id"])) + except TypeError: + Logger.logException("w", "Local material {} has invalid values.".format(material["id"])) + + return result diff --git a/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py b/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py new file mode 100644 index 0000000000..b669eb192a --- /dev/null +++ b/plugins/UM3NetworkPrinting/tests/TestSendMaterialJob.py @@ -0,0 +1,190 @@ +# Copyright (c) 2018 Ultimaker B.V. +# Copyright (c) 2018 Ultimaker B.V. +# Cura is released under the terms of the LGPLv3 or higher. +import io +import json +from unittest import TestCase, mock +from unittest.mock import patch, call + +from PyQt5.QtCore import QByteArray + +from UM.MimeTypeDatabase import MimeType +from UM.Application import Application +from plugins.UM3NetworkPrinting.src.SendMaterialJob import SendMaterialJob + + +@patch("builtins.open", lambda _, __: io.StringIO("")) +@patch("UM.MimeTypeDatabase.MimeTypeDatabase.getMimeTypeForFile", + lambda _: MimeType(name = "application/x-ultimaker-material-profile", comment = "Ultimaker Material Profile", + suffixes = ["xml.fdm_material"])) +@patch("UM.Resources.Resources.getAllResourcesOfType", lambda _: ["/materials/generic_pla_white.xml.fdm_material"]) +@patch("plugins.UM3NetworkPrinting.src.ClusterUM3OutputDevice") +@patch("PyQt5.QtNetwork.QNetworkReply") +class TestSendMaterialJob(TestCase): + _LOCAL_MATERIAL_WHITE = {"type": "material", "status": "unknown", "id": "generic_pla_white", + "base_file": "generic_pla_white", "setting_version": "5", "name": "White PLA", + "brand": "Generic", "material": "PLA", "color_name": "White", + "GUID": "badb0ee7-87c8-4f3f-9398-938587b67dce", "version": "1", "color_code": "#ffffff", + "description": "Test PLA White", "adhesion_info": "Use glue.", "approximate_diameter": "3", + "properties": {"density": "1.00", "diameter": "2.85", "weight": "750"}, + "definition": "fdmprinter", "compatible": True} + + _LOCAL_MATERIAL_BLACK = {"type": "material", "status": "unknown", "id": "generic_pla_black", + "base_file": "generic_pla_black", "setting_version": "5", "name": "Yellow CPE", + "brand": "Ultimaker", "material": "CPE", "color_name": "Black", + "GUID": "5fbb362a-41f9-4818-bb43-15ea6df34aa4", "version": "1", "color_code": "#000000", + "description": "Test PLA Black", "adhesion_info": "Use glue.", "approximate_diameter": "3", + "properties": {"density": "1.01", "diameter": "2.85", "weight": "750"}, + "definition": "fdmprinter", "compatible": True} + + _REMOTE_MATERIAL_WHITE = { + "guid": "badb0ee7-87c8-4f3f-9398-938587b67dce", + "material": "PLA", + "brand": "Generic", + "version": 1, + "color": "White", + "density": 1.00 + } + + _REMOTE_MATERIAL_BLACK = { + "guid": "5fbb362a-41f9-4818-bb43-15ea6df34aa4", + "material": "PLA", + "brand": "Generic", + "version": 2, + "color": "Black", + "density": 1.00 + } + + def test_run(self, device_mock, reply_mock): + job = SendMaterialJob(device_mock) + job.run() + + # We expect the materials endpoint to be called when the job runs. + device_mock.get.assert_called_with("materials/", on_finished = job._onGetRemoteMaterials) + + def test__onGetRemoteMaterials_withFailedRequest(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 404 + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # We expect the device not to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + def test__onGetRemoteMaterials_withWrongEncoding(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("cp500")) + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # Given that the parsing fails we do no expect the device to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + def test__onGetRemoteMaterials_withBadJsonAnswer(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(b"Six sick hicks nick six slick bricks with picks and sticks.") + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # Given that the parsing fails we do no expect the device to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + def test__onGetRemoteMaterials_withMissingGuidInRemoteMaterial(self, reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + remote_material_without_guid = self._REMOTE_MATERIAL_WHITE.copy() + del remote_material_without_guid["guid"] + reply_mock.readAll.return_value = QByteArray(json.dumps([remote_material_without_guid]).encode("ascii")) + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + # Given that parsing fails we do not expect the device to be called for any follow up. + self.assertEqual(0, device_mock.createFormPart.call_count) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withInvalidVersionInLocalMaterial(self, application_mock, container_registry_mock, + reply_mock, device_mock): + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) + + localMaterialWhiteWithInvalidVersion = self._LOCAL_MATERIAL_WHITE.copy() + localMaterialWhiteWithInvalidVersion["version"] = "one" + container_registry_mock.findContainersMetadata.return_value = [localMaterialWhiteWithInvalidVersion] + + application_mock.getContainerRegistry.return_value = container_registry_mock + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(0, device_mock.createFormPart.call_count) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withNoUpdate(self, application_mock, container_registry_mock, reply_mock, + device_mock): + application_mock.getContainerRegistry.return_value = container_registry_mock + + device_mock.createFormPart.return_value = "_xXx_" + + container_registry_mock.findContainersMetadata.return_value = [self._LOCAL_MATERIAL_WHITE] + + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(0, device_mock.createFormPart.call_count) + self.assertEqual(0, device_mock.postFormWithParts.call_count) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withUpdatedMaterial(self, application_mock, container_registry_mock, reply_mock, + device_mock): + application_mock.getContainerRegistry.return_value = container_registry_mock + + device_mock.createFormPart.return_value = "_xXx_" + + localMaterialWhiteWithHigherVersion = self._LOCAL_MATERIAL_WHITE.copy() + localMaterialWhiteWithHigherVersion["version"] = "2" + container_registry_mock.findContainersMetadata.return_value = [localMaterialWhiteWithHigherVersion] + + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_WHITE]).encode("ascii")) + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(1, device_mock.createFormPart.call_count) + self.assertEqual(1, device_mock.postFormWithParts.call_count) + self.assertEquals( + [call.createFormPart("name=\"file\"; filename=\"generic_pla_white.xml.fdm_material\"", ""), + call.postFormWithParts(target = "materials/", parts = ["_xXx_"], on_finished = job.sendingFinished)], + device_mock.method_calls) + + @patch("cura.Settings.CuraContainerRegistry") + @patch("UM.Application") + def test__onGetRemoteMaterials_withNewMaterial(self, application_mock, container_registry_mock, reply_mock, + device_mock): + application_mock.getContainerRegistry.return_value = container_registry_mock + + device_mock.createFormPart.return_value = "_xXx_" + + container_registry_mock.findContainersMetadata.return_value = [self._LOCAL_MATERIAL_WHITE, + self._LOCAL_MATERIAL_BLACK] + + reply_mock.attribute.return_value = 200 + reply_mock.readAll.return_value = QByteArray(json.dumps([self._REMOTE_MATERIAL_BLACK]).encode("ascii")) + + with mock.patch.object(Application, "getInstance", new = lambda: application_mock): + job = SendMaterialJob(device_mock) + job._onGetRemoteMaterials(reply_mock) + + self.assertEqual(1, device_mock.createFormPart.call_count) + self.assertEqual(1, device_mock.postFormWithParts.call_count) + self.assertEquals( + [call.createFormPart("name=\"file\"; filename=\"generic_pla_white.xml.fdm_material\"", ""), + call.postFormWithParts(target = "materials/", parts = ["_xXx_"], on_finished = job.sendingFinished)], + device_mock.method_calls)