From 1d528c205f433850f7c7889c7d5750ef5ec4b732 Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Wed, 27 Nov 2024 12:00:03 -0700 Subject: [PATCH 1/6] look for modules in extensions directory. --- aqt/archives.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/aqt/archives.py b/aqt/archives.py index 29c6b0d0..e94b051d 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -31,6 +31,10 @@ from aqt.helper import Settings, get_hash, getUrl, ssplit from aqt.metadata import QtRepoProperty, Version +@dataclass +class UpdateXmls: + target_folder: str + xml_text: str @dataclass class TargetConfig: @@ -354,6 +358,7 @@ def _target_packages(self) -> ModuleToPackage: package_names = [ f"qt.qt{self.version.major}.{self._version_str()}.{suffix}", f"qt.{self._version_str()}.{suffix}", + f"extensions.{module}.{self._version_str()}.{self.arch}", ] if not module.startswith("addons."): package_names.append(f"qt.qt{self.version.major}.{self._version_str()}.addons.{suffix}") @@ -399,7 +404,24 @@ def _get_archives_base(self, name, target_packages): ) update_xml_url = posixpath.join(os_target_folder, "Updates.xml") update_xml_text = self._download_update_xml(update_xml_url) - self._parse_update_xml(os_target_folder, update_xml_text, target_packages) + arch = self.arch + if self.os_name == "windows": + arch = self.arch.replace("win64_", "", 1) + elif self.os_name == "linux": + arch = "x86_64" + elif self.os_name == "linux_arm64": + arch = "arm64" + extensions_target_folder = posixpath.join( + "online/qtsdkrepository", + os_name, + "extensions/qtwebengine", + self._version_str(), + arch + ) + extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml") + extensions_xml_text = self._download_update_xml(extensions_xml_url) + update_xmls = [UpdateXmls(os_target_folder, update_xml_text), UpdateXmls(extensions_target_folder, extensions_xml_text) ] + self._parse_update_xmls(update_xmls, target_packages) def _download_update_xml(self, update_xml_path): """Hook for unit test.""" @@ -451,6 +473,10 @@ def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages: pkg_update_name=packageupdate.name, # For testing purposes ) ) + + def _parse_update_xmls(self, update_xmls, target_packages: Optional[ModuleToPackage]): + for update_xml in update_xmls: + self._parse_update_xml(update_xml.target_folder, update_xml.xml_text, target_packages) # if we have located every requested package, then target_packages will be empty if not self.all_extra and len(target_packages) > 0: message = f"The packages {target_packages} were not found while parsing XML of package information!" From aa222a59dbcca231a4950f2c9c42817edf6b2186 Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:12:29 -0700 Subject: [PATCH 2/6] search a list of possible extensions. currently qtwebengine, qtpdf. --- aqt/archives.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/aqt/archives.py b/aqt/archives.py index e94b051d..e1197e3a 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -404,6 +404,7 @@ def _get_archives_base(self, name, target_packages): ) update_xml_url = posixpath.join(os_target_folder, "Updates.xml") update_xml_text = self._download_update_xml(update_xml_url) + update_xmls = [UpdateXmls(os_target_folder, update_xml_text)] arch = self.arch if self.os_name == "windows": arch = self.arch.replace("win64_", "", 1) @@ -411,16 +412,18 @@ def _get_archives_base(self, name, target_packages): arch = "x86_64" elif self.os_name == "linux_arm64": arch = "arm64" - extensions_target_folder = posixpath.join( - "online/qtsdkrepository", - os_name, - "extensions/qtwebengine", - self._version_str(), - arch - ) - extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml") - extensions_xml_text = self._download_update_xml(extensions_xml_url) - update_xmls = [UpdateXmls(os_target_folder, update_xml_text), UpdateXmls(extensions_target_folder, extensions_xml_text) ] + for ext in ["qtwebengine", "qtpdf"]: + extensions_target_folder = posixpath.join( + "online/qtsdkrepository", + os_name, + "extensions", + ext, + self._version_str(), + arch + ) + extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml") + extensions_xml_text = self._download_update_xml(extensions_xml_url) + update_xmls.append(UpdateXmls(extensions_target_folder, extensions_xml_text)) self._parse_update_xmls(update_xmls, target_packages) def _download_update_xml(self, update_xml_path): From 5a13230a3b2fc0666502f11b1c5b3cf21303b175 Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:17:27 -0700 Subject: [PATCH 3/6] fix pre 6.8.0 installs --- aqt/archives.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/aqt/archives.py b/aqt/archives.py index e1197e3a..f3f56d34 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -405,25 +405,28 @@ def _get_archives_base(self, name, target_packages): update_xml_url = posixpath.join(os_target_folder, "Updates.xml") update_xml_text = self._download_update_xml(update_xml_url) update_xmls = [UpdateXmls(os_target_folder, update_xml_text)] - arch = self.arch - if self.os_name == "windows": - arch = self.arch.replace("win64_", "", 1) - elif self.os_name == "linux": - arch = "x86_64" - elif self.os_name == "linux_arm64": - arch = "arm64" - for ext in ["qtwebengine", "qtpdf"]: - extensions_target_folder = posixpath.join( - "online/qtsdkrepository", - os_name, - "extensions", - ext, - self._version_str(), - arch - ) - extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml") - extensions_xml_text = self._download_update_xml(extensions_xml_url) - update_xmls.append(UpdateXmls(extensions_target_folder, extensions_xml_text)) + + if self.version >= Version("6.8.0"): + arch = self.arch + if self.os_name == "windows": + arch = self.arch.replace("win64_", "", 1) + elif self.os_name == "linux": + arch = "x86_64" + elif self.os_name == "linux_arm64": + arch = "arm64" + for ext in ["qtwebengine", "qtpdf"]: + extensions_target_folder = posixpath.join( + "online/qtsdkrepository", + os_name, + "extensions", + ext, + self._version_str(), + arch + ) + extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml") + extensions_xml_text = self._download_update_xml(extensions_xml_url) + update_xmls.append(UpdateXmls(extensions_target_folder, extensions_xml_text)) + self._parse_update_xmls(update_xmls, target_packages) def _download_update_xml(self, update_xml_path): From 83010648686f25b1cf3e289ecbbbf9b211618c21 Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:22:56 -0700 Subject: [PATCH 4/6] get --modules and --long-modules working with 6.8.0+ extensions --- aqt/metadata.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/aqt/metadata.py b/aqt/metadata.py index 4750537d..70e4fe30 100644 --- a/aqt/metadata.py +++ b/aqt/metadata.py @@ -231,14 +231,39 @@ def is_qt(self) -> bool: def is_tools(self) -> bool: return self.category == "tools" - def to_url(self) -> str: - return "online/qtsdkrepository/{os}{arch}/{target}/".format( + def to_os_arch(self): + return "{os}{arch}".format( os=self.host, arch=( "_x86" if self.host == "windows" else ("" if self.host in ("linux_arm64", "all_os", "windows_arm64") else "_x64") ), + ) + + def to_extension_folder(self, module, version, arch) -> str: + extarch = arch + if self.host == "windows": + extarch = arch.replace("win64_", "", 1) + elif self.host == "linux": + extarch = "x86_64" + elif self.host == "linux_arm64": + extarch = "arm64" + return "online/qtsdkrepository/{osarch}/extensions/{ext}/{ver}/{extarch}/".format( + osarch=self.to_os_arch(), + ext=module, + ver=version, + extarch=extarch, + ) + + def to_extension_url(self) -> str: + return "online/qtsdkrepository/{osarch}/extensions/".format( + osarch=self.to_os_arch(), + ) + + def to_url(self) -> str: + return "online/qtsdkrepository/{osarch}/{target}/".format( + osarch=self.to_os_arch(), target=self.target, ) @@ -667,6 +692,10 @@ def filter_by(ver: Version, ext: str) -> bool: def fetch_latest_version(self, ext: str) -> Optional[Version]: return self.fetch_versions(ext).latest() + def fetch_extensions(self) -> List[str]: + html_doc = self.fetch_http(self.archive_id.to_extension_url(), False) + return list(self.iterate_folders(html_doc, self.base_url)) + def fetch_tools(self) -> List[str]: html_doc = self.fetch_http(self.archive_id.to_url(), False) return list(self.iterate_folders(html_doc, self.base_url, filter_category="tools")) @@ -824,6 +853,14 @@ def _fetch_module_metadata(self, folder: str, predicate: Optional[Callable[[Elem predicate=predicate if predicate else MetadataFactory._has_nonempty_downloads, ) + def _fetch_extension_metadata(self, url: str, predicate: Optional[Callable[[Element], bool]] = None): + rest_of_url = posixpath.join(url, "Updates.xml") + xml = self.fetch_http(rest_of_url) if not Settings.ignore_hash else self.fetch_http(rest_of_url, False) + return xml_to_modules( + xml, + predicate=predicate if predicate else MetadataFactory._has_nonempty_downloads, + ) + def fetch_modules(self, version: Version, arch: str) -> List[str]: """Returns list of modules""" extension = QtRepoProperty.extension_for_arch(arch, version >= Version("6.0.0")) @@ -849,6 +886,9 @@ def to_module_arch(name: str) -> Tuple[Optional[str], Optional[str]]: module, _arch = to_module_arch(name) if _arch == arch: modules.add(cast(str, module)) + if version >= Version("6.8.0"): + for ext in self.fetch_extensions(): + modules.add(ext) return sorted(modules) @staticmethod @@ -863,6 +903,8 @@ def fetch_long_modules(self, version: Version, arch: str) -> ModuleData: extension = QtRepoProperty.extension_for_arch(arch, version >= Version("6.0.0")) qt_ver_str = self._get_qt_version_str(version) # Example: re.compile(r"^(preview\.)?qt\.(qt5\.)?590(\.addons)?\.(?P[^.]+)\.gcc_64$") + # qt.qt6.680.addons.qtwebsockets.win64_msvc2022_64 + # qt.qt6.680.debug_info.win64_msvc2022_64 pattern = re.compile( r"^(preview\.)?qt\.(qt" + str(version.major) @@ -885,6 +927,25 @@ def matches_arch(element: Element) -> bool: if module is not None: m[module] = value + # Examples: extensions.qtwebengine.680.debug_information + # extensions.qtwebengine.680.win64_msvc2022_64 + ext_pattern = re.compile( + r"^extensions\." + + r"(?P[^.]+)\." + + qt_ver_str + + r"\." + + arch + + r"$" + ) + if version >= Version("6.8.0"): + for ext in self.fetch_extensions(): + ext_meta = self._fetch_extension_metadata(self.archive_id.to_extension_folder(ext, qt_ver_str, arch)) + for key, value in ext_meta.items(): + ext_match = ext_pattern.match(key) + if ext_match is not None: + module = ext_match.group("module") + if module is not None: + m[module] = value return ModuleData(m) def fetch_modules_sde(self, cmd_type: str, version: Version) -> List[str]: From ee40b25e840cc09ae5e5af140692a6c4ca6a09d8 Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:28:56 -0700 Subject: [PATCH 5/6] fix regression failures. --- aqt/archives.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aqt/archives.py b/aqt/archives.py index f3f56d34..413473e7 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -481,6 +481,8 @@ def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages: ) def _parse_update_xmls(self, update_xmls, target_packages: Optional[ModuleToPackage]): + if not target_packages: + target_packages = ModuleToPackage({}) for update_xml in update_xmls: self._parse_update_xml(update_xml.target_folder, update_xml.xml_text, target_packages) # if we have located every requested package, then target_packages will be empty From a1ffb938a8b80b8832b86c45aa2f9bfca691263d Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:43:16 -0700 Subject: [PATCH 6/6] fix regression "check packaging" --- aqt/archives.py | 11 ++++------- aqt/metadata.py | 11 ++--------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/aqt/archives.py b/aqt/archives.py index 413473e7..7c4dcf1c 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -31,11 +31,13 @@ from aqt.helper import Settings, get_hash, getUrl, ssplit from aqt.metadata import QtRepoProperty, Version + @dataclass class UpdateXmls: target_folder: str xml_text: str + @dataclass class TargetConfig: version: str @@ -409,19 +411,14 @@ def _get_archives_base(self, name, target_packages): if self.version >= Version("6.8.0"): arch = self.arch if self.os_name == "windows": - arch = self.arch.replace("win64_", "", 1) + arch = self.arch.replace("win64_", "", 1) elif self.os_name == "linux": arch = "x86_64" elif self.os_name == "linux_arm64": arch = "arm64" for ext in ["qtwebengine", "qtpdf"]: extensions_target_folder = posixpath.join( - "online/qtsdkrepository", - os_name, - "extensions", - ext, - self._version_str(), - arch + "online/qtsdkrepository", os_name, "extensions", ext, self._version_str(), arch ) extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml") extensions_xml_text = self._download_update_xml(extensions_xml_url) diff --git a/aqt/metadata.py b/aqt/metadata.py index 70e4fe30..f1f563e7 100644 --- a/aqt/metadata.py +++ b/aqt/metadata.py @@ -244,7 +244,7 @@ def to_os_arch(self): def to_extension_folder(self, module, version, arch) -> str: extarch = arch if self.host == "windows": - extarch = arch.replace("win64_", "", 1) + extarch = arch.replace("win64_", "", 1) elif self.host == "linux": extarch = "x86_64" elif self.host == "linux_arm64": @@ -929,14 +929,7 @@ def matches_arch(element: Element) -> bool: # Examples: extensions.qtwebengine.680.debug_information # extensions.qtwebengine.680.win64_msvc2022_64 - ext_pattern = re.compile( - r"^extensions\." - + r"(?P[^.]+)\." - + qt_ver_str - + r"\." - + arch - + r"$" - ) + ext_pattern = re.compile(r"^extensions\." + r"(?P[^.]+)\." + qt_ver_str + r"\." + arch + r"$") if version >= Version("6.8.0"): for ext in self.fetch_extensions(): ext_meta = self._fetch_extension_metadata(self.archive_id.to_extension_folder(ext, qt_ver_str, arch))