Merge pull request #19798 from Ultimaker/CURA-12229_blacklist_binaries

Refactor binaries filtering and update dependencies

conandata.yml

@@ -124,6 +124,19 @@ pyinstaller:
         Windows: "./icons/Cura.ico"
         Macos: "./icons/cura.icns"
         Linux: "./icons/cura-128.png"
+    blacklist:
+      - [ "assimp" ]
+      - [ "qt", "charts" ]
+      - [ "qt", "coap" ]
+      - [ "qt", "data", "vis" ]
+      - [ "qt", "lab", "animat" ]
+      - [ "qt", "mqtt" ]
+      - [ "qt", "net", "auth" ]
+      - [ "qt", "quick3d" ]
+      - [ "qt", "timeline" ]
+      - [ "qt", "virt", "key" ]
+      - [ "qt", "wayland", "compos" ]
+      - [ "qt", "5", "compat" ]
 pycharm_targets:
   - jinja_path: .run_templates/pycharm_cura_run.run.xml.jinja
     module_name: Cura

conanfile.py

@@ -217,6 +217,62 @@ class CuraConan(ConanFile):
                 python_installs=self._python_installs(),
             ))
 
+    def _delete_unwanted_binaries(self, root):
+        dynamic_binary_file_exts = [".so", ".dylib", ".dll", ".pyd", ".pyi"]
+        prohibited = [
+            "qt5compat",
+            "qtcharts",
+            "qtcoap",
+            "qtdatavis3d",
+            "qtlottie",
+            "qtmqtt",
+            "qtnetworkauth",
+            "qtquick3d",
+            "qtquick3dphysics",
+            "qtquicktimeline",
+            "qtvirtualkeyboard",
+            "qtwayland"
+        ]
+        forbiddens = [x.encode() for x in prohibited]
+        to_remove_files = []
+        to_remove_dirs = []
+        for root, dir_, files in os.walk(root):
+            for filename in files:
+                if not any([(x in filename) for x in dynamic_binary_file_exts]):
+                    continue
+                pathname = os.path.join(root, filename)
+                still_exist = True
+                for forbidden in prohibited:
+                    if forbidden.lower() in str(pathname).lower():
+                        to_remove_files.append(pathname)
+                        still_exist = False
+                        break
+                if not still_exist:
+                    continue
+                with open(pathname, "rb") as file:
+                    bytez = file.read().lower()
+                    for forbidden in forbiddens:
+                        if bytez.find(forbidden) >= 0:
+                            to_remove_files.append(pathname)
+            for dirname in dir_:
+                for forbidden in prohibited:
+                    if forbidden.lower() == str(dirname).lower():
+                        pathname = os.path.join(root, dirname)
+                        to_remove_dirs.append(pathname)
+                        break
+        for file in to_remove_files:
+            try:
+                os.remove(file)
+                print(f"deleted file: {file}")
+            except Exception as ex:
+                print(f"WARNING: Attempt to delete file {file} results in: {str(ex)}")
+        for dir_ in to_remove_dirs:
+            try:
+                rmdir(self, dir_)
+                print(f"deleted dir_: {dir_}")
+            except Exception as ex:
+                print(f"WARNING: Attempt to delete folder {dir_} results in: {str(ex)}")
+
     def _generate_pyinstaller_spec(self, location, entrypoint_location, icon_path, entitlements_file):
         pyinstaller_metadata = self.conan_data["pyinstaller"]
         datas = []
@@ -281,13 +337,27 @@ class CuraConan(ConanFile):
         version = self.conf.get("user.cura:version", default = self.version, check_type = str)
         cura_version = Version(version)
 
+        # filter all binary files in binaries on the blacklist
+        blacklist = pyinstaller_metadata["blacklist"]
+        filtered_binaries = [b for b in binaries if not any([all([(part in b[0].lower()) for part in parts]) for parts in blacklist])]
+
+        # In case the installer isn't actually pyinstaller (Windows at the moment), outright remove the offending files:
+        specifically_delete = set(binaries) - set(filtered_binaries)
+        for (unwanted_path, _) in specifically_delete:
+            try:
+                os.remove(unwanted_path)
+                print(f"delete: {unwanted_path}")
+            except Exception as ex:
+                print(f"WARNING: Attempt to delete binary {unwanted_path} results in: {str(ex)}")
+
+        # Write the actual file:
         with open(os.path.join(location, "UltiMaker-Cura.spec"), "w") as f:
             f.write(pyinstaller.render(
                 name = str(self.options.display_name).replace(" ", "-"),
                 display_name = self._app_name,
                 entrypoint = entrypoint_location,
                 datas = datas,
-                binaries = binaries,
+                binaries = filtered_binaries,
                 venv_script_path = str(self._script_dir),
                 hiddenimports = pyinstaller_metadata["hiddenimports"],
                 collect_all = pyinstaller_metadata["collect_all"],
@@ -405,8 +475,10 @@ class CuraConan(ConanFile):
 
         for dependency in self.dependencies.host.values():
             for bindir in dependency.cpp_info.bindirs:
+                self._delete_unwanted_binaries(bindir)
                 copy(self, "*.dll", bindir, str(self._site_packages), keep_path = False)
             for libdir in dependency.cpp_info.libdirs:
+                self._delete_unwanted_binaries(libdir)
                 copy(self, "*.pyd", libdir, str(self._site_packages), keep_path = False)
                 copy(self, "*.pyi", libdir, str(self._site_packages), keep_path = False)
                 copy(self, "*.dylib", libdir, str(self._base_dir.joinpath("lib")), keep_path = False)
@@ -496,6 +568,11 @@ echo "CURA_APP_NAME={{ cura_app_name }}" >> ${{ env_prefix }}GITHUB_ENV
 
         self._generate_cura_version(os.path.join(self._site_packages, "cura"))
 
+        self._delete_unwanted_binaries(self._site_packages)
+        self._delete_unwanted_binaries(self.package_folder)
+        self._delete_unwanted_binaries(self._base_dir)
+        self._delete_unwanted_binaries(self._share_dir)
+
         entitlements_file = "'{}'".format(Path(self.cpp_info.res_paths[2], "MacOS", "cura.entitlements"))
         self._generate_pyinstaller_spec(location = self._base_dir,
                                         entrypoint_location = "'{}'".format(os.path.join(self.package_folder, self.cpp_info.bindirs[0], self.conan_data["pyinstaller"]["runinfo"]["entrypoint"])).replace("\\", "\\\\"),

requirements.txt

@@ -31,16 +31,6 @@ PyQt6-Qt6==6.6.0 \
     --hash=sha256:8cb30d64a4d32465ea1686bc827cbe452225fb387c4873356b0fa7b9ae63534f \
     --hash=sha256:a151f34712cd645111e89cb30b02e5fb69c9dcc3603ab3c03a561e874bd7cbcf \
     --hash=sha256:e5483ae04bf107411c7469f1be9f9e2eb9840303e788b3ac524fe30af90d45f4
-PyQt6-NetworkAuth==6.6.0 \
-    --hash=sha256:7b90b81792fe53105287c8cbb5e4b22bc44a482268ffb7d3e33f852807f86182 \
-    --hash=sha256:c7e2335159aa795e2fe6fb069ccce6308672ab80f26c50fab57caf957371cbb5 \
-    --hash=sha256:cdfc0bfaea16a9e09f075bdafefb996aa9fdec392052ba4fb3cbac233c1958fb \
-    --hash=sha256:f60ff9a62f5129dc2a9d4c495fb47f9a03e4dfb666b50fb7d61f46e89bf7b6a2
-PyQt6-NetworkAuth-Qt6==6.6.0 \
-    --hash=sha256:481d9093e1fb1ac6843d8beabcd359cc34b74b9a2cbb3e2b68d96bd3f178d4e0 \
-    --hash=sha256:4cc48fd375730a0ba5fbed9d64abb2914f587377560a78a63aff893f9e276a45 \
-    --hash=sha256:5006deabf55304d4a3e0b3c954f93e5835546b11e789d14653a2493d12d3a063 \
-    --hash=sha256:bcd56bfc892fec961c51eba3c0bf32ba8317a762d9e254d3830569611ed569d6
 
 certifi==2023.5.7; \
     --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716