From 5dd6700031ac5cb102aa3fee2bfee540f395a0d9 Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Wed, 23 Oct 2024 17:08:15 +0200 Subject: [PATCH 1/8] Refactor binaries filtering and update dependencies Introduce a mechanism to filter blacklisted binaries from the build configuration and update the conandata.yml with a new blacklist section. Additionally, remove unnecessary dependencies from requirements.txt, particularly entries for PyQt6-NetworkAuth and related packages. Contribute to CURA-12229 --- conandata.yml | 7 +++++++ conanfile.py | 6 +++++- requirements.txt | 10 ---------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/conandata.yml b/conandata.yml index fa0305261c..b3a71a4b50 100644 --- a/conandata.yml +++ b/conandata.yml @@ -124,6 +124,13 @@ pyinstaller: Windows: "./icons/Cura.ico" Macos: "./icons/cura.icns" Linux: "./icons/cura-128.png" + blacklist: + - "libassimp.so" + - "PyQt6/Qt6/qml/QtQuick3D" # directory + - "libqtquick3deffectplugin.so" + - "libqtquick3dphysicshelpersplugin.so" + - "libqtquick3dparticleeffectsplugin.so" + - "etc......." pycharm_targets: - jinja_path: .run_templates/pycharm_cura_run.run.xml.jinja module_name: Cura diff --git a/conanfile.py b/conanfile.py index 999cca3362..be515c876d 100644 --- a/conanfile.py +++ b/conanfile.py @@ -281,13 +281,17 @@ class CuraConan(ConanFile): version = self.conf.get("user.cura:version", default = self.version, check_type = str) cura_version = Version(version) + # filter all binary files in binaries on the blacklist + blacklist = pyinstaller_metadata["blacklist"] + filtered_binaries = [b for b in binaries if b not in blacklist] + with open(os.path.join(location, "UltiMaker-Cura.spec"), "w") as f: f.write(pyinstaller.render( name = str(self.options.display_name).replace(" ", "-"), display_name = self._app_name, entrypoint = entrypoint_location, datas = datas, - binaries = binaries, + binaries = filtered_binaries, venv_script_path = str(self._script_dir), hiddenimports = pyinstaller_metadata["hiddenimports"], collect_all = pyinstaller_metadata["collect_all"], diff --git a/requirements.txt b/requirements.txt index 344a0156d8..83da635cd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,16 +31,6 @@ PyQt6-Qt6==6.6.0 \ --hash=sha256:8cb30d64a4d32465ea1686bc827cbe452225fb387c4873356b0fa7b9ae63534f \ --hash=sha256:a151f34712cd645111e89cb30b02e5fb69c9dcc3603ab3c03a561e874bd7cbcf \ --hash=sha256:e5483ae04bf107411c7469f1be9f9e2eb9840303e788b3ac524fe30af90d45f4 -PyQt6-NetworkAuth==6.6.0 \ - --hash=sha256:7b90b81792fe53105287c8cbb5e4b22bc44a482268ffb7d3e33f852807f86182 \ - --hash=sha256:c7e2335159aa795e2fe6fb069ccce6308672ab80f26c50fab57caf957371cbb5 \ - --hash=sha256:cdfc0bfaea16a9e09f075bdafefb996aa9fdec392052ba4fb3cbac233c1958fb \ - --hash=sha256:f60ff9a62f5129dc2a9d4c495fb47f9a03e4dfb666b50fb7d61f46e89bf7b6a2 -PyQt6-NetworkAuth-Qt6==6.6.0 \ - --hash=sha256:481d9093e1fb1ac6843d8beabcd359cc34b74b9a2cbb3e2b68d96bd3f178d4e0 \ - --hash=sha256:4cc48fd375730a0ba5fbed9d64abb2914f587377560a78a63aff893f9e276a45 \ - --hash=sha256:5006deabf55304d4a3e0b3c954f93e5835546b11e789d14653a2493d12d3a063 \ - --hash=sha256:bcd56bfc892fec961c51eba3c0bf32ba8317a762d9e254d3830569611ed569d6 certifi==2023.5.7; \ --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 From 52824c487272e5f28e9ac0f7b126aca19008b5de Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 29 Oct 2024 13:18:04 +0100 Subject: [PATCH 2/8] Build: Make pseudo-code into something that should work for all OS's. part of CURA-12229 --- conandata.yml | 18 ++++++++++++------ conanfile.py | 7 ++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/conandata.yml b/conandata.yml index b3a71a4b50..735e1ce9e0 100644 --- a/conandata.yml +++ b/conandata.yml @@ -125,12 +125,18 @@ pyinstaller: Macos: "./icons/cura.icns" Linux: "./icons/cura-128.png" blacklist: - - "libassimp.so" - - "PyQt6/Qt6/qml/QtQuick3D" # directory - - "libqtquick3deffectplugin.so" - - "libqtquick3dphysicshelpersplugin.so" - - "libqtquick3dparticleeffectsplugin.so" - - "etc......." + - [ "assimp" ] + - [ "qt", "charts" ] + - [ "qt", "coap" ] + - [ "qt", "data", "vis" ] + - [ "qt", "lab", "animat" ] + - [ "qt", "mqtt" ] + - [ "qt", "net", "auth" ] + - [ "qt", "quick3d" ] + - [ "qt", "timeline" ] + - [ "qt", "virt", "key" ] + - [ "qt", "wayland", "compos" ] + - [ "qt", "5", "compat" ] pycharm_targets: - jinja_path: .run_templates/pycharm_cura_run.run.xml.jinja module_name: Cura diff --git a/conanfile.py b/conanfile.py index be515c876d..9461ba6c1d 100644 --- a/conanfile.py +++ b/conanfile.py @@ -283,7 +283,12 @@ class CuraConan(ConanFile): # filter all binary files in binaries on the blacklist blacklist = pyinstaller_metadata["blacklist"] - filtered_binaries = [b for b in binaries if b not in blacklist] + filtered_binaries = [b for b in binaries if any([all([(part in b) for part in parts]) for parts in blacklist])] + + # TEMP: print to make sure -- remove this before merge + print("specifically don't include:") + print(set(binaries) - set(filtered_binaries)) + print("=== === ===") with open(os.path.join(location, "UltiMaker-Cura.spec"), "w") as f: f.write(pyinstaller.render( From 2d9e7a347ffbf8468a838c92fbb5b6d9784ac631 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 29 Oct 2024 13:49:24 +0100 Subject: [PATCH 3/8] Build: Fix inverted filter condition for libraries. Noticed that we _only_ kept the libraries we _didn't_ want, instead of the inverse :-p part of CURA-12229 --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 9461ba6c1d..0062431f34 100644 --- a/conanfile.py +++ b/conanfile.py @@ -283,7 +283,7 @@ class CuraConan(ConanFile): # filter all binary files in binaries on the blacklist blacklist = pyinstaller_metadata["blacklist"] - filtered_binaries = [b for b in binaries if any([all([(part in b) for part in parts]) for parts in blacklist])] + filtered_binaries = [b for b in binaries if not any([all([(part in b) for part in parts]) for parts in blacklist])] # TEMP: print to make sure -- remove this before merge print("specifically don't include:") From e0f0e9407b6e5f092b69a85e16fde5484f64ab0c Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 29 Oct 2024 16:14:19 +0100 Subject: [PATCH 4/8] Build: Filter didn't work because the filename was in a tuple. part of CURA-12229 --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 0062431f34..d684e10ea4 100644 --- a/conanfile.py +++ b/conanfile.py @@ -283,7 +283,7 @@ class CuraConan(ConanFile): # filter all binary files in binaries on the blacklist blacklist = pyinstaller_metadata["blacklist"] - filtered_binaries = [b for b in binaries if not any([all([(part in b) for part in parts]) for parts in blacklist])] + filtered_binaries = [b for b in binaries if not any([all([(part in b[0].lower()) for part in parts]) for parts in blacklist])] # TEMP: print to make sure -- remove this before merge print("specifically don't include:") From cf836c29cb47ffb784cd965f50508fa415903b36 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 30 Oct 2024 08:45:58 +0100 Subject: [PATCH 5/8] Build: Outright remove the unwanted binaries, just to be sure. This is needed on Windows at the moment, which doesn't actually parse that file, but does execute the method. part of CURA-12229 --- conanfile.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conanfile.py b/conanfile.py index d684e10ea4..d2ecd6e3e4 100644 --- a/conanfile.py +++ b/conanfile.py @@ -285,11 +285,13 @@ class CuraConan(ConanFile): blacklist = pyinstaller_metadata["blacklist"] filtered_binaries = [b for b in binaries if not any([all([(part in b[0].lower()) for part in parts]) for parts in blacklist])] - # TEMP: print to make sure -- remove this before merge - print("specifically don't include:") - print(set(binaries) - set(filtered_binaries)) - print("=== === ===") + # In case the installer isn't actually pyinstaller (Windows at the moment), outright remove the offending files: + specifically_delete = set(binaries) - set(filtered_binaries) + for (unwanted_path, _) in specifically_delete: + print(f"delete: {unwanted_path}") + os.remove(unwanted_path) + # Write the actual file: with open(os.path.join(location, "UltiMaker-Cura.spec"), "w") as f: f.write(pyinstaller.render( name = str(self.options.display_name).replace(" ", "-"), From 2aed655c8b238b17dc879cb8ae7326e15ba0832a Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 30 Oct 2024 15:07:00 +0100 Subject: [PATCH 6/8] Build: Really try to make sure unwanted binaries are gone. part of CURA-12229 --- conanfile.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/conanfile.py b/conanfile.py index d2ecd6e3e4..3b24d6b991 100644 --- a/conanfile.py +++ b/conanfile.py @@ -217,6 +217,56 @@ class CuraConan(ConanFile): python_installs=self._python_installs(), )) + def _delete_unwanted_binaries(self, root): + dynamic_binary_file_exts = [".so", ".dylib", ".dll", ".pyd", ".pyi"] + prohibited = [ + "qt5compat", + "qtcharts", + "qtcoap", + "qtdatavis3d", + "qtlottie", + "qtmqtt", + "qtnetworkauth", + "qtquick3d", + "qtquick3dphysics", + "qtquicktimeline", + "qtvirtualkeyboard", + "qtwayland" + ] + forbiddens = [x.encode() for x in prohibited] + to_remove_files = [] + to_remove_dirs = [] + for root, dir_, files in os.walk(root): + for filename in files: + if not any([(x in filename) for x in dynamic_binary_file_exts]): + continue + pathname = os.path.join(root, filename) + still_exist = True + for forbidden in prohibited: + if forbidden.lower() in str(pathname).lower(): + to_remove_files.append(pathname) + still_exist = False + break + if not still_exist: + continue + with open(pathname, "rb") as file: + bytez = file.read().lower() + for forbidden in forbiddens: + if bytez.find(forbidden) >= 0: + to_remove_files.append(pathname) + for dirname in dir_: + for forbidden in prohibited: + if forbidden.lower() == str(dirname).lower(): + pathname = os.path.join(root, dirname) + to_remove_dirs.append(pathname) + break + for file in to_remove_files: + os.remove(file) + print(f"deleted file: {file}") + for dir_ in to_remove_dirs: + os.remove(dir_) + print(f"deleted dir_: {dir_}") + def _generate_pyinstaller_spec(self, location, entrypoint_location, icon_path, entitlements_file): pyinstaller_metadata = self.conan_data["pyinstaller"] datas = [] @@ -416,8 +466,10 @@ class CuraConan(ConanFile): for dependency in self.dependencies.host.values(): for bindir in dependency.cpp_info.bindirs: + self._delete_unwanted_binaries(bindir) copy(self, "*.dll", bindir, str(self._site_packages), keep_path = False) for libdir in dependency.cpp_info.libdirs: + self._delete_unwanted_binaries(libdir) copy(self, "*.pyd", libdir, str(self._site_packages), keep_path = False) copy(self, "*.pyi", libdir, str(self._site_packages), keep_path = False) copy(self, "*.dylib", libdir, str(self._base_dir.joinpath("lib")), keep_path = False) @@ -507,6 +559,11 @@ echo "CURA_APP_NAME={{ cura_app_name }}" >> ${{ env_prefix }}GITHUB_ENV self._generate_cura_version(os.path.join(self._site_packages, "cura")) + self._delete_unwanted_binaries(self._site_packages) + self._delete_unwanted_binaries(self.package_folder) + self._delete_unwanted_binaries(self._base_dir) + self._delete_unwanted_binaries(self._share_dir) + entitlements_file = "'{}'".format(Path(self.cpp_info.res_paths[2], "MacOS", "cura.entitlements")) self._generate_pyinstaller_spec(location = self._base_dir, entrypoint_location = "'{}'".format(os.path.join(self.package_folder, self.cpp_info.bindirs[0], self.conan_data["pyinstaller"]["runinfo"]["entrypoint"])).replace("\\", "\\\\"), From cd2e4220f0c4616a7e15fc9673a691a16f599d83 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 30 Oct 2024 15:49:02 +0100 Subject: [PATCH 7/8] Build: Don't remove dir with os.remove use rmdir instead. part of CURA-12229 --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 3b24d6b991..1f86867ae8 100644 --- a/conanfile.py +++ b/conanfile.py @@ -264,7 +264,7 @@ class CuraConan(ConanFile): os.remove(file) print(f"deleted file: {file}") for dir_ in to_remove_dirs: - os.remove(dir_) + rmdir(self, dir_) print(f"deleted dir_: {dir_}") def _generate_pyinstaller_spec(self, location, entrypoint_location, icon_path, entitlements_file): From 86f12f62f72d2cbc252386b50fc87ba3ef9b260d Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 30 Oct 2024 17:02:42 +0100 Subject: [PATCH 8/8] Build: Some files may have already been removed. Symlinks for example? part of CURA-12229 --- conanfile.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/conanfile.py b/conanfile.py index 1f86867ae8..73336e41f3 100644 --- a/conanfile.py +++ b/conanfile.py @@ -261,11 +261,17 @@ class CuraConan(ConanFile): to_remove_dirs.append(pathname) break for file in to_remove_files: - os.remove(file) - print(f"deleted file: {file}") + try: + os.remove(file) + print(f"deleted file: {file}") + except Exception as ex: + print(f"WARNING: Attempt to delete file {file} results in: {str(ex)}") for dir_ in to_remove_dirs: - rmdir(self, dir_) - print(f"deleted dir_: {dir_}") + try: + rmdir(self, dir_) + print(f"deleted dir_: {dir_}") + except Exception as ex: + print(f"WARNING: Attempt to delete folder {dir_} results in: {str(ex)}") def _generate_pyinstaller_spec(self, location, entrypoint_location, icon_path, entitlements_file): pyinstaller_metadata = self.conan_data["pyinstaller"] @@ -338,8 +344,11 @@ class CuraConan(ConanFile): # In case the installer isn't actually pyinstaller (Windows at the moment), outright remove the offending files: specifically_delete = set(binaries) - set(filtered_binaries) for (unwanted_path, _) in specifically_delete: - print(f"delete: {unwanted_path}") - os.remove(unwanted_path) + try: + os.remove(unwanted_path) + print(f"delete: {unwanted_path}") + except Exception as ex: + print(f"WARNING: Attempt to delete binary {unwanted_path} results in: {str(ex)}") # Write the actual file: with open(os.path.join(location, "UltiMaker-Cura.spec"), "w") as f: