refactor: deb_import

apt/BUILD.bazel

@@ -34,5 +34,6 @@ bzl_library(
         "//apt/private:deb_resolve",
         "//apt/private:deb_translate_lock",
         "//apt/private:lockfile",
+        "//apt/private:manifest",
     ],
 )

apt/apt.bzl

@@ -114,8 +114,8 @@ def _apt_install(
                 and avoid the DEBUG messages.
         package_template: (EXPERIMENTAL!) a template file for generated BUILD
                           files. Available template replacement keys are:
-                          `{target_name}`, `{deps}`, `{urls}`, `{name}`,
-                          `{arch}`, `{sha256}`, `{repo_name}`
+                          `{target_name}`, `{src}`, `{deps}`, `{urls}`,
+                          `{name}`, `{arch}`, `{sha256}`, `{repo_name}`
         resolve_transitive: whether dependencies of dependencies should be
                             resolved and added to the lockfile.
     """
@@ -126,8 +126,10 @@ def _apt_install(
     )
 
     if not lock and not nolock:
-        # buildifier: disable=print
-        print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % name)
+        print(
+            "\nNo lockfile was given. To create one please run " +
+            "`bazel run @{}//:lock`".format(name),
+        )
 
     _deb_translate_lock(
         name = name,

apt/extensions.bzl

@@ -1,9 +1,10 @@
 "apt extensions"
 
-load("//apt/private:deb_import.bzl", "deb_import")
-load("//apt/private:deb_resolve.bzl", "deb_resolve", "internal_resolve")
+load("//apt/private:deb_import.bzl", "deb_import", "make_deb_import_key")
+load("//apt/private:deb_resolve.bzl", "deb_resolve")
 load("//apt/private:deb_translate_lock.bzl", "deb_translate_lock")
 load("//apt/private:lockfile.bzl", "lockfile")
+load("//apt/private:manifest.bzl", "manifest")
 
 def _distroless_extension(module_ctx):
     root_direct_deps = []
@@ -13,30 +14,27 @@ def _distroless_extension(module_ctx):
         for install in mod.tags.install:
             lockf = None
             if not install.lock:
-                lockf = internal_resolve(
+                lockf = manifest.lock(
                     module_ctx,
-                    "yq",
                     install.manifest,
                     install.resolve_transitive,
                 )
 
                 if not install.nolock:
-                    # buildifier: disable=print
-                    print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % install.name)
+                    print(
+                        "\nNo lockfile was given. To create one please run " +
+                        "`bazel run @{}//:lock`".format(install.name),
+                    )
             else:
                 lockf = lockfile.from_json(module_ctx, module_ctx.read(install.lock))
 
-            for (package) in lockf.packages():
-                package_key = lockfile.make_package_key(
-                    package["name"],
-                    package["version"],
-                    package["arch"],
-                )
+            for package in lockf.packages():
+                deb_import_key = make_deb_import_key(install.name, package)
 
                 deb_import(
-                    name = "%s_%s" % (install.name, package_key),
-                    urls = [package["url"]],
-                    sha256 = package["sha256"],
+                    name = deb_import_key,
+                    url = package.url,
+                    sha256 = package.sha256,
                 )
 
             deb_resolve(

apt/private/BUILD.bazel

@@ -26,6 +26,7 @@ bzl_library(
     srcs = ["deb_translate_lock.bzl"],
     visibility = ["//apt:__subpackages__"],
     deps = [
+        ":deb_import",
         ":lockfile",
         "@bazel_tools//tools/build_defs/repo:cache.bzl",
         "@bazel_tools//tools/build_defs/repo:http.bzl",
@@ -37,21 +38,45 @@ bzl_library(
     name = "lockfile",
     srcs = ["lockfile.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = [":util"],
+    deps = [":pkg"],
+)
+
+bzl_library(
+    name = "pkg",
+    srcs = ["pkg.bzl"],
+    visibility = ["//apt:__subpackages__"],
+)
+
+bzl_library(
+    name = "manifest",
+    srcs = ["manifest.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [
+        ":apt_deb_repository",
+        ":apt_dep_resolver",
+        ":lockfile",
+        ":util",
+        ":version_constraint",
+        "@aspect_bazel_lib//lib:repo_utils",
+    ],
 )
 
 bzl_library(
     name = "apt_deb_repository",
     srcs = ["apt_deb_repository.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = [":util"],
+    deps = [
+        ":nested_dict",
+        ":version_constraint",
+    ],
 )
 
 bzl_library(
     name = "apt_dep_resolver",
     srcs = ["apt_dep_resolver.bzl"],
     visibility = ["//apt:__subpackages__"],
     deps = [
+        ":util",
         ":version",
         ":version_constraint",
     ],
@@ -65,6 +90,7 @@ bzl_library(
         ":apt_deb_repository",
         ":apt_dep_resolver",
         ":lockfile",
+        ":manifest",
         "@aspect_bazel_lib//lib:repo_utils",
     ],
 )
@@ -80,13 +106,20 @@ bzl_library(
     name = "deb_import",
     srcs = ["deb_import.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = ["@bazel_tools//tools/build_defs/repo:http.bzl"],
+    deps = [
+        ":starlark_codegen",
+        ":util",
+        "@bazel_tools//tools/build_defs/repo:http.bzl",
+    ],
 )
 
 bzl_library(
     name = "util",
     srcs = ["util.bzl"],
     visibility = ["//apt:__subpackages__"],
+    deps = [
+        "@bazel_skylib//lib:sets",
+    ],
 )
 
 bzl_library(
@@ -95,3 +128,18 @@ bzl_library(
     visibility = ["//apt:__subpackages__"],
     deps = [":version"],
 )
+
+bzl_library(
+    name = "nested_dict",
+    srcs = ["nested_dict.bzl"],
+    visibility = ["//apt:__subpackages__"],
+)
+
+bzl_library(
+    name = "starlark_codegen",
+    srcs = ["starlark_codegen.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [
+        ":util",
+    ],
+)

apt/private/apt_deb_repository.bzl

@@ -1,60 +1,68 @@
 "https://wiki.debian.org/DebianRepository"
 
-load(":util.bzl", "util")
+load(":nested_dict.bzl", "nested_dict")
 load(":version_constraint.bzl", "version_constraint")
 
-def _fetch_package_index(rctx, url, dist, comp, arch, integrity):
-    target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch)
-
+def _fetch_package_index(rctx, source):
     # See https://linux.die.net/man/1/xz and https://linux.die.net/man/1/gzip
     #  --keep       -> keep the original file (Bazel might be still committing the output to the cache)
     #  --force      -> overwrite the output if it exists
     #  --decompress -> decompress
     supported_extensions = {
         "xz": ["xz", "--decompress", "--keep", "--force"],
         "gz": ["gzip", "--decompress", "--keep", "--force"],
+        "": None,
     }
 
     failed_attempts = []
 
-    for (ext, cmd) in supported_extensions.items():
-        output = "{}/Packages.{}".format(target_triple, ext)
-        dist_url = "{}/dists/{}/{}/binary-{}/Packages.{}".format(url, dist, comp, arch, ext)
+    for ext, cmd in supported_extensions.items():
+        index_url = source.index_url(ext)
+        output_full = source.output_full(ext)
+
         download = rctx.download(
-            url = dist_url,
-            output = output,
-            integrity = integrity,
+            url = index_url,
+            output = output_full,
             allow_fail = True,
         )
-        decompress_r = None
-        if download.success:
-            decompress_r = rctx.execute(cmd + [output])
-            if decompress_r.return_code == 0:
-                integrity = download.integrity
-                break
 
-        failed_attempts.append((dist_url, download, decompress_r))
+        if not download.success:
+            reason = "Download failed. See warning above for details."
+            failed_attempts.append((index_url, reason))
+            continue
 
-    if len(failed_attempts) == len(supported_extensions):
-        attempt_messages = []
-        for (url, download, decompress) in failed_attempts:
-            reason = "unknown"
-            if not download.success:
-                reason = "Download failed. See warning above for details."
-            elif decompress.return_code != 0:
-                reason = "Decompression failed with non-zero exit code.\n\n{}\n{}".format(decompress.stderr, decompress.stdout)
+        if cmd == None:
+            # index is already decompressed
+            break
+
+        decompress_cmd = cmd + [output_full]
+        decompress_res = rctx.execute(decompress_cmd)
+
+        if decompress_res.return_code == 0:
+            break
 
-            attempt_messages.append("""\n*) Failed '{}'\n\n{}""".format(url, reason))
+        reason = "'{cmd}' returned a non-zero exit code: {return_code}"
+        reason += "\n\n{stderr}\n{stdout}"
+        reason = reason.format(
+            cmd = decompress_cmd,
+            return_code = decompress_res.return_code,
+            stderr = decompress_res.stderr,
+            stdout = decompress_res.stdout,
+        )
+
+        failed_attempts.append((index_url, reason))
 
-        fail("""
-** Tried to download {} different package indices and all failed. 
+    if len(failed_attempts) == len(supported_extensions):
+        attempt_messages = [
+            "\n  * '{}' FAILED:\n\n  {}".format(url, reason)
+            for url, reason in failed_attempts
+        ]
 
-{}
-        """.format(len(failed_attempts), "\n".join(attempt_messages)))
+        fail("Failed to fetch packages index:\n" + "\n".join(attempt_messages))
 
-    return ("{}/Packages".format(target_triple), integrity)
+    return rctx.read(source.output)
 
-def _parse_repository(state, contents, root):
+def _parse_package_index(state, contents, source):
     last_key = ""
     pkg = {}
     for group in contents.split("\n\n"):
@@ -83,78 +91,51 @@ def _parse_repository(state, contents, root):
             pkg[key] = value
 
         if len(pkg.keys()) != 0:
-            pkg["Root"] = root
+            pkg["Root"] = source.base_url
             _add_package(state, pkg)
             last_key = ""
             pkg = {}
 
 def _add_package(state, package):
-    util.set_dict(state.packages, value = package, keys = (package["Architecture"], package["Package"], package["Version"]))
+    state.packages.set(
+        keys = (package["Architecture"], package["Package"], package["Version"]),
+        value = package,
+    )
 
     # https://www.debian.org/doc/debian-policy/ch-relationships.html#virtual-packages-provides
     if "Provides" in package:
-        provides = version_constraint.parse_dep(package["Provides"])
-        vp = util.get_dict(state.virtual_packages, (package["Architecture"], provides["name"]), [])
-        vp.append((provides, package))
-        util.set_dict(state.virtual_packages, vp, (package["Architecture"], provides["name"]))
-
-def _virtual_packages(state, name, arch):
-    return util.get_dict(state.virtual_packages, [arch, name], [])
+        provides = version_constraint.parse_provides(package["Provides"])
 
-def _package_versions(state, name, arch):
-    return util.get_dict(state.packages, [arch, name], {}).keys()
-
-def _package(state, name, version, arch):
-    return util.get_dict(state.packages, keys = (arch, name, version))
+        state.virtual_packages.add(
+            keys = (package["Architecture"], provides["name"]),
+            value = (provides["version"], package),
+        )
 
-def _create(rctx, sources, archs):
+def _new(rctx, manifest):
     state = struct(
-        packages = dict(),
-        virtual_packages = dict(),
+        packages = nested_dict.new(),
+        virtual_packages = nested_dict.new(),
     )
 
-    for arch in archs:
-        for (url, dist, comp) in sources:
-            # We assume that `url` does not contain a trailing forward slash when passing to
-            # functions below. If one is present, remove it. Some HTTP servers do not handle
-            # redirects properly when a path contains "//"
-            # (ie. https://mymirror.com/ubuntu//dists/noble/stable/... may return a 404
-            # on misconfigured HTTP servers)
-            url = url.rstrip("/")
+    for source in manifest.sources:
+        index = "%s/%s" % (source.index_path, source.index)
 
-            rctx.report_progress("Fetching package index: {}/{} for {}".format(dist, comp, arch))
-            (output, _) = _fetch_package_index(rctx, url, dist, comp, arch, "")
+        rctx.report_progress("Fetching package index: %s" % index)
+        output = _fetch_package_index(rctx, source)
 
-            # TODO: this is expensive to perform.
-            rctx.report_progress("Parsing package index: {}/{} for {}".format(dist, comp, arch))
-            _parse_repository(state, rctx.read(output), url)
+        rctx.report_progress("Parsing package index: %s" % index)
+        _parse_package_index(state, output, source)
 
     return struct(
-        package_versions = lambda **kwargs: _package_versions(state, **kwargs),
-        virtual_packages = lambda **kwargs: _virtual_packages(state, **kwargs),
-        package = lambda **kwargs: _package(state, **kwargs),
+        package_versions = lambda arch, name: state.packages.get((arch, name), {}).keys(),
+        virtual_packages = lambda arch, name: state.virtual_packages.get((arch, name), []),
+        package = lambda arch, name, version: state.packages.get((arch, name, version)),
     )
 
 deb_repository = struct(
-    new = _create,
-)
-
-# TESTONLY: DO NOT DEPEND ON THIS
-def _create_test_only():
-    state = struct(
-        packages = dict(),
-        virtual_packages = dict(),
-    )
-
-    return struct(
-        package_versions = lambda **kwargs: _package_versions(state, **kwargs),
-        virtual_packages = lambda **kwargs: _virtual_packages(state, **kwargs),
-        package = lambda **kwargs: _package(state, **kwargs),
-        parse_repository = lambda contents: _parse_repository(state, contents, "http://nowhere"),
-        packages = state.packages,
-        reset = lambda: state.packages.clear(),
-    )
-
-DO_NOT_DEPEND_ON_THIS_TEST_ONLY = struct(
-    new = _create_test_only,
+    new = _new,
+    __test__ = struct(
+        _fetch_package_index = _fetch_package_index,
+        _parse_package_index = _parse_package_index,
+    ),
 )