fix: NVIDIA CUDA flat repos don't follow Debian repo spec

Although the Debian repo spec for 'Filename' (see
https://wiki.debian.org/DebianRepository/Format#Filename) clearly says
that 'Filename' should be relative to the base directory of the repo and
should be in canonical form (i.e. without '.' or '..') there are cases
where this is not honored.

In those cases we try to work around this by assuming 'Filename' is
relative to the sources.list directory/ so we combine them and normalize
the new 'Filename' path.

Note that, so far, only the NVIDIA CUDA repos needed this workaround so
maybe this heuristic will break for other repos that don't conform to
the Debian repo spec.

MODULE.bazel

@@ -6,7 +6,7 @@ module(
     compatibility_level = 1,
 )
 
-bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
 bazel_dep(name = "aspect_bazel_lib", version = "2.7.9")
 
 bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
@@ -21,7 +21,7 @@ use_repo(bazel_lib_toolchains, "yq_linux_s390x")
 use_repo(bazel_lib_toolchains, "yq_windows_amd64")
 
 bazel_dep(name = "gazelle", version = "0.34.0", dev_dependency = True, repo_name = "bazel_gazelle")
-bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True)
+bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True)
 bazel_dep(name = "buildifier_prebuilt", version = "6.1.2", dev_dependency = True)
 bazel_dep(name = "platforms", version = "0.0.10", dev_dependency = True)
 bazel_dep(name = "rules_oci", version = "2.0.0-rc0", dev_dependency = True)

WORKSPACE.bazel

@@ -57,6 +57,17 @@ load("@bullseye_rproject//:packages.bzl", "bullseye_rproject_packages")
 
 bullseye_rproject_packages()
 
+# bazel run @nvidia_ubuntu2404_cuda//:lock
+deb_index(
+    name = "nvidia_ubuntu2404_cuda",
+    lock = "//examples/debian_flat_repo:nvidia_ubuntu2404_cuda.lock.json",
+    manifest = "//examples/debian_flat_repo:nvidia_ubuntu2404_cuda.yaml",
+)
+
+load("@nvidia_ubuntu2404_cuda//:packages.bzl", "nvidia_ubuntu2404_cuda_packages")
+
+nvidia_ubuntu2404_cuda_packages()
+
 deb_index(
     name = "apt_security",
     manifest = "//examples/debian_snapshot_security:security.yaml",

apt/private/BUILD.bazel

@@ -60,7 +60,10 @@ bzl_library(
     name = "package_index",
     srcs = ["package_index.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = [":version"],
+    deps = [
+        ":version",
+        "@bazel_skylib//lib:paths",
+    ],
 )
 
 bzl_library(

apt/private/package_index.bzl

@@ -1,5 +1,7 @@
 "package index"
 
+load("@bazel_skylib//lib:paths.bzl", "paths")
+load(":util.bzl", "util")
 load(":version.bzl", version_lib = "version")
 
 def _fetch_package_index(rctx, source):
@@ -56,6 +58,40 @@ def _fetch_package_index(rctx, source):
 
     return rctx.read(source.output)
 
+def _make_file_url(pkg, source):
+    filename = pkg["Filename"]
+
+    invalid_filename = not paths.is_normalized(
+        filename,
+        look_for_same_level_references = True,
+    )
+
+    if invalid_filename:
+        # NOTE:
+        # Although the Debian repo spec for 'Filename' (see
+        # https://wiki.debian.org/DebianRepository/Format#Filename) clearly
+        # says that 'Filename' should be relative to the base directory of the
+        # repo and should be in canonical form (i.e. without '.' or '..') there
+        # are cases where this is not honored.
+        #
+        # In those cases we try to work around this by assuming 'Filename' is
+        # relative to the sources.list "index path" (e.g. directory/ for flat
+        # repos) so we combine them and normalize the new 'Filename' path.
+        #
+        # Note that, so far, only the NVIDIA CUDA repos needed this workaround
+        # so maybe this heuristic will break for other repos that don't conform
+        # to the Debian repo spec.
+        filename = paths.normalize(paths.join(source.index_path, filename))
+
+    base_url = util.parse_url(source.base_url)
+    file_url = "{}://{}{}".format(
+        base_url.scheme,
+        base_url.host,
+        paths.join(base_url.path, filename),
+    )
+
+    return file_url, invalid_filename
+
 def _package_set(packages, keys, package):
     for key in keys[:-1]:
         if key not in packages:
@@ -66,6 +102,9 @@ def _package_set(packages, keys, package):
 def _parse_package_index(packages, contents, source):
     last_key = ""
     pkg = {}
+    total_pkgs = 0
+    out_of_spec = []
+
     for group in contents.split("\n\n"):
         for line in group.split("\n"):
             if line.strip() == "":
@@ -92,7 +131,10 @@ def _parse_package_index(packages, contents, source):
             pkg[key] = value
 
         if len(pkg.keys()) != 0:
-            pkg["Root"] = source.base_url
+            pkg["FileUrl"], invalid_filename = _make_file_url(pkg, source)
+
+            if invalid_filename:
+                out_of_spec.append(pkg["Package"])
 
             # NOTE: workaround for multi-arch flat repos
             arch = source.arch if pkg["Architecture"] == "all" else pkg["Architecture"]
@@ -104,6 +146,9 @@ def _parse_package_index(packages, contents, source):
             )
             last_key = ""
             pkg = {}
+            total_pkgs += 1
+
+    return out_of_spec, total_pkgs
 
 def _package_get(packages, arch, name, version = None):
     versions = packages.get(arch, {}).get(name, {})
@@ -123,7 +168,13 @@ def _index(rctx, manifest):
         output = _fetch_package_index(rctx, source)
 
         rctx.report_progress("Parsing package index: %s" % index)
-        _parse_package_index(packages, output, source)
+        out_of_spec, total_pkgs = _parse_package_index(packages, output, source)
+
+        if out_of_spec:
+            count = len(out_of_spec)
+            pct = int(100.0 * count / total_pkgs)
+            msg = "Warning: index {} has {} packages ({}%) with invalid 'Filename' fields"
+            print(msg.format(index, count, pct))
 
     return struct(
         packages = packages,
@@ -287,6 +338,7 @@ package_index = struct(
     parse_depends = _parse_depends,
     # NOTE: these are exposed here for testing purposes, DO NOT USE OTHERWISE
     _fetch_package_index = _fetch_package_index,
+    _make_file_url = _make_file_url,
     _parse_package_index = _parse_package_index,
     _package_set = _package_set,
     _package_get = _package_get,

apt/private/pkg.bzl

@@ -4,7 +4,7 @@ def _pkg_from_index(package, arch):
     return struct(
         name = package["Package"],
         version = package["Version"],
-        url = "%s/%s" % (package["Root"], package["Filename"]),
+        url = package["FileUrl"],
         sha256 = package["SHA256"],
         arch = arch,
         dependencies = [],

apt/private/util.bzl

@@ -11,10 +11,27 @@ def _get_dupes(list_):
 
     return dupes
 
+def _parse_url(url):
+    if "://" not in url:
+        fail("Invalid URL: %s" % url)
+
+    scheme, url_ = url.split("://", 1)
+
+    path = "/"
+
+    if "/" in url_:
+        host, path_ = url_.split("/", 1)
+        path += path_
+    else:
+        host = url_
+
+    return struct(scheme = scheme, host = host, path = path)
+
 def _sanitize(str):
     return str.replace("+", "-p-").replace(":", "-").replace("~", "_")
 
 util = struct(
     get_dupes = _get_dupes,
+    parse_url = _parse_url,
     sanitize = _sanitize,
 )

apt/tests/BUILD.bazel

@@ -1,9 +1,12 @@
 load(":manifest_test.bzl", "manifest_tests")
 load(":package_index_test.bzl", "package_index_tests")
+load(":util_test.bzl", "util_tests")
 load(":version_test.bzl", "version_tests")
 
 manifest_tests()
 
 package_index_tests()
 
+util_tests()
+
 version_tests()

apt/tests/package_index_test.bzl

@@ -50,7 +50,7 @@ def _parse_package_index_test(ctx):
     package_index._parse_package_index(actual, output, source)
 
     asserts.equals(env, "foo", actual[arch][name][version]["Package"])
-    asserts.equals(env, url, actual[arch][name][version]["Root"])
+    asserts.true(env, actual[arch][name][version]["FileUrl"].startswith(url))
 
     return unittest.end(env)
 
@@ -107,6 +107,8 @@ def _index_test(ctx):
 
     url = "http://mirror.com"
 
+    source = mock.manifest(url, arch, name).sources[0]
+
     mock_rctx = mock.rctx(
         read = mock.read(output),
         download = mock.download(success = True),
@@ -116,10 +118,17 @@ def _index_test(ctx):
     actual = package_index._index(mock_rctx, mock.manifest(url, arch, name))
 
     expected_pkg = mock.pkg(arch, name, version)
-    expected_pkg["Root"] = url
+    file_url, _ = package_index._make_file_url(expected_pkg, source)
+    expected_pkg["FileUrl"] = file_url
 
     actual_pkg = actual.package_get(arch, name, version)
-    asserts.equals(env, expected_pkg, actual_pkg)
+
+    # NOTE: we compare key-by-key because the error output of
+    # asserts.equals(env, expected_pkg, actual_pkg) is quite
+    # hard to read
+    asserts.equals(env, expected_pkg.keys(), actual_pkg.keys())
+    for key in expected_pkg.keys():
+        asserts.equals(env, expected_pkg[key], actual_pkg[key])
 
     expected_packages = {arch: {name: {version: expected_pkg}}}
     asserts.equals(env, expected_packages, actual.packages)

apt/tests/util_test.bzl

@@ -0,0 +1,25 @@
+"unit tests for util"
+
+load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest")
+load("//apt/private:util.bzl", "util")
+
+_TEST_SUITE_PREFIX = "util/"
+
+def _parse_url_test(ctx):
+    env = unittest.begin(ctx)
+
+    parameters = {
+        "https://mirror.com": struct(scheme = "https", host = "mirror.com", path = "/"),
+        "http://mirror.com/foo/bar": struct(scheme = "http", host = "mirror.com", path = "/foo/bar"),
+    }
+
+    for url, expected in parameters.items():
+        actual = util.parse_url(url)
+        asserts.equals(env, expected, actual)
+
+    return unittest.end(env)
+
+parse_url_test = unittest.make(_parse_url_test)
+
+def util_tests():
+    parse_url_test(name = _TEST_SUITE_PREFIX + "parse_url")

examples/debian_flat_repo/BUILD.bazel

@@ -5,28 +5,51 @@ load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load")
 PACKAGES = [
     "@bullseye//dpkg",
     "@bullseye//apt",
+]
+
+PACKAGES_AMD64 = PACKAGES + [
     "@bullseye_rproject//r-mathlib",
+    "@nvidia_ubuntu2404_cuda//nvidia-container-toolkit-base",
+]
+
+PACKAGES_ARM64 = PACKAGES + [
+    "@nvidia_ubuntu2404_cuda//nvidia-container-toolkit-base",
 ]
 
 # Creates /var/lib/dpkg/status with installed package information.
 dpkg_status(
     name = "dpkg_status",
-    controls = [
-        "%s/amd64:control" % package
-        for package in PACKAGES
-    ],
+    controls = select({
+        "@platforms//cpu:x86_64": [
+            "%s/amd64:control" % package
+            for package in PACKAGES_AMD64
+        ],
+        "@platforms//cpu:arm64": [
+            "%s/arm64:control" % package
+            for package in PACKAGES_ARM64
+        ],
+    }),
 )
 
 oci_image(
     name = "apt",
-    architecture = "amd64",
+    architecture = select({
+        "@platforms//cpu:x86_64": "amd64",
+        "@platforms//cpu:arm64": "arm64",
+    }),
     os = "linux",
     tars = [
         ":dpkg_status",
-    ] + [
-        "%s/amd64" % package
-        for package in PACKAGES
-    ],
+    ] + select({
+        "@platforms//cpu:x86_64": [
+            "%s/amd64" % package
+            for package in PACKAGES_AMD64
+        ],
+        "@platforms//cpu:arm64": [
+            "%s/arm64" % package
+            for package in PACKAGES_ARM64
+        ],
+    }),
 )
 
 oci_load(
@@ -39,10 +62,15 @@ oci_load(
 
 container_structure_test(
     name = "test",
-    configs = ["test_linux_amd64.yaml"],
+    configs = select({
+        "@platforms//cpu:x86_64": ["test_linux_amd64.yaml"],
+        "@platforms//cpu:arm64": ["test_linux_arm64.yaml"],
+    }),
     image = ":apt",
-    target_compatible_with = [
-        "@platforms//cpu:x86_64",
+    target_compatible_with = select({
+        "@platforms//cpu:x86_64": ["@platforms//cpu:x86_64"],
+        "@platforms//cpu:arm64": ["@platforms//cpu:arm64"],
+    }) + [
         "@platforms//os:linux",
     ],
 )