feat: implement an apt resolver

version lib

more progress

statusd

docs

.bazelrc

@@ -21,6 +21,10 @@ build --java_runtime_version=remotejdk_11
 # https://github.com/GoogleContainerTools/rules_distroless/actions/runs/7118944984/job/19382981899?pr=9#step:8:51
 common:linux --sandbox_tmpfs_path=/tmp
 
+
+# Allow external dependencies to be retried. debian snapshot is unreliable and needs retries.
+common --experimental_repository_downloader_retries=200
+
 # Load any settings specific to the current user.
 # .bazelrc.user should appear in .gitignore so that settings are not shared with team members
 # This needs to be last statement in this

MODULE.bazel

@@ -8,10 +8,18 @@ module(
 
 bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "aspect_bazel_lib", version = "2.5.3")
+bazel_dep(name = "container_structure_test", version = "1.16.0")
+bazel_dep(name = "rules_oci", version = "1.7.4")
 
 bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
-bazel_lib_toolchains.tar()
 use_repo(bazel_lib_toolchains, "bsd_tar_toolchains")
+use_repo(bazel_lib_toolchains, "yq_darwin_amd64")
+use_repo(bazel_lib_toolchains, "yq_darwin_arm64")
+use_repo(bazel_lib_toolchains, "yq_linux_amd64")
+use_repo(bazel_lib_toolchains, "yq_linux_arm64")
+use_repo(bazel_lib_toolchains, "yq_linux_ppc64le")
+use_repo(bazel_lib_toolchains, "yq_linux_s390x")
+use_repo(bazel_lib_toolchains, "yq_windows_amd64")
 
 bazel_dep(name = "gazelle", version = "0.34.0", dev_dependency = True, repo_name = "bazel_gazelle")
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True)

WORKSPACE.bazel

@@ -4,7 +4,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "example-bullseye-ca-certificates",
-    build_file_content = 'exports_files(["data.tar.xz"])',
+    build_file_content = 'exports_files(["data.tar.xz", "control.tar.xz"])',
     sha256 = "b2d488ad4d8d8adb3ba319fc9cb2cf9909fc42cb82ad239a26c570a2e749c389",
     urls = ["https://snapshot.debian.org/archive/debian/20231106T210201Z/pool/main/c/ca-certificates/ca-certificates_20210119_all.deb"],
 )
@@ -22,3 +22,16 @@ http_archive(
     sha256 = "38c44247c5b3e864d6db2877edd9c9a0555fc4e23ae271b73d7f527802616df5",
     urls = ["https://snapshot.debian.org/archive/debian-security/20231106T230332Z/pool/updates/main/g/glibc/libc-bin_2.36-9+deb12u3_armhf.deb"],
 )
+
+load("//apt:index.bzl", "deb_index")
+
+# bazel run @bullseye//:lock
+deb_index(
+    name = "bullseye",
+    lock = "//examples/apt:bullseye.lock.json",
+    manifest = "//examples/apt:bullseye.yaml",
+)
+
+load("@bullseye//:packages.bzl", "bullseye_packages")
+
+bullseye_packages()

apt/BUILD.bazel

@@ -0,0 +1,25 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+exports_files([
+    "index.bzl",
+])
+
+bzl_library(
+    name = "defs",
+    srcs = ["defs.bzl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//apt/private:dpkg_status",
+        "//apt/private:dpkg_statusd",
+    ],
+)
+
+bzl_library(
+    name = "index",
+    srcs = ["index.bzl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//apt/private:index",
+        "//apt/private:resolve",
+    ],
+)

apt/defs.bzl

@@ -0,0 +1,7 @@
+"EXPERIMENTAL! Public API"
+
+load("//apt/private:dpkg_status.bzl", _dpkg_status = "dpkg_status")
+load("//apt/private:dpkg_statusd.bzl", _dpkg_statusd = "dpkg_statusd")
+
+dpkg_status = _dpkg_status
+dpkg_statusd = _dpkg_statusd

apt/index.bzl

@@ -0,0 +1,79 @@
+"apt-get"
+
+load("//apt/private:index.bzl", _deb_package_index = "deb_package_index")
+load("//apt/private:resolve.bzl", _deb_resolve = "deb_resolve")
+
+def deb_index(
+        name,
+        manifest,
+        lock = None,
+        package_template = None,
+        resolve_transitive = True):
+    """A convience repository macro around package_index and resolve repository rules.
+
+    WORKSPACE example;
+
+    ```starlark
+    load("@rules_distroless//apt:index.bzl", "deb_index")
+
+    deb_index(
+        name = "bullseye",
+        # For the initial setup, the lockfile attribute can be omitted  and generated by running
+        #    bazel run @bullseye//:lock
+        # This will generate the lock.json file next to the manifest file by replacing `.yaml` with `.lock.json`
+        lock = "//examples/apt:bullseye.lock.json",
+        manifest = "//examples/apt:bullseye.yaml",
+    )
+
+    load("@bullseye//:packages.bzl", "bullseye_packages")
+    bullseye_packages()
+    ```
+
+    BZLMOD example;
+    ```starlark
+    # TODO: support BZLMOD
+    ```
+
+    This macro will expand to two repositories;  `<name>` and `<name>_resolve`.
+
+    A typical workflow for `deb_index` involves generation of a lockfile `deb_resolve`
+    and consumption of lockfile by `deb_package_index` for generating a DAG.
+
+    The lockfile generation can be `on-demand` by omitting the lock attribute, however,
+    this comes with the cost of doing a new package resolution on repository cache misses.
+
+    While we strongly encourage users to check in the generated lockfile, it's not always
+    possible to check in the generated lockfile as by default Debian repositories are rolling,
+    therefore a lockfile generated today might not work work tomorrow  as the upstream
+    repository might publish new version of a package.
+
+    That said, users can still use a `debian archive snapshot` repository and check-in the
+    generated lockfiles. This is possible because by design `debian snapshot` repositories
+    are immutable point-in-time snapshot of the upstream repositories, which means packages
+    never get deleted or updated in a specific snapshot.
+
+    An example of this could be found [here](/examples/apt).
+
+    Args:
+        name: name of the repository
+        manifest: label to a `manifest.yaml`
+        lock: label to a `lock.json`
+        package_template: (EXPERIMENTAL!) a template string for generated BUILD files.
+          Available template replacement keys are: `{target_name}`, `{deps}`, `{urls}`, `{name}`, `{arch}`, `{sha256}`, `{repo_name}`
+        resolve_transitive: whether dependencies of dependencies should be resolved and added to the lockfile.
+    """
+    _deb_resolve(
+        name = name + "_resolution",
+        manifest = manifest,
+        resolve_transitive = resolve_transitive,
+    )
+
+    if not lock:
+        # buildifier: disable=print
+        print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % name)
+
+    _deb_package_index(
+        name = name,
+        lock = lock if lock else "@" + name + "_resolution//:lock.json",
+        package_template = package_template,
+    )

apt/private/BUILD.bazel

@@ -0,0 +1,73 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+exports_files([
+    "dpkg_statusd.sh",
+    "dpkg_status.sh",
+])
+
+bzl_library(
+    name = "dpkg_status",
+    srcs = ["dpkg_status.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = ["//distroless/private:tar"],
+)
+
+bzl_library(
+    name = "dpkg_statusd",
+    srcs = ["dpkg_statusd.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = ["//distroless/private:tar"],
+)
+
+bzl_library(
+    name = "index",
+    srcs = ["index.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [":lockfile"],
+)
+
+bzl_library(
+    name = "lockfile",
+    srcs = ["lockfile.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [":util"],
+)
+
+bzl_library(
+    name = "package_index",
+    srcs = ["package_index.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [":util"],
+)
+
+bzl_library(
+    name = "package_resolution",
+    srcs = ["package_resolution.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [":version"],
+)
+
+bzl_library(
+    name = "resolve",
+    srcs = ["resolve.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [
+        ":lockfile",
+        ":package_index",
+        ":package_resolution",
+        "@aspect_bazel_lib//lib:repo_utils",
+    ],
+)
+
+bzl_library(
+    name = "version",
+    srcs = ["version.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = ["@aspect_bazel_lib//lib:strings"],
+)
+
+bzl_library(
+    name = "util",
+    srcs = ["util.bzl"],
+    visibility = ["//apt:__subpackages__"],
+)

apt/private/dpkg_status.bzl

@@ -0,0 +1,46 @@
+"dpkg_status"
+
+# buildifier: disable=bzl-visibility
+load("//distroless/private:tar.bzl", "tar_lib")
+
+_DOC = """TODO: docs"""
+
+def _dpkg_status_impl(ctx):
+    bsdtar = ctx.toolchains[tar_lib.TOOLCHAIN_TYPE]
+
+    output = ctx.actions.declare_file(ctx.attr.name + ".tar")
+
+    args = ctx.actions.args()
+    args.add(bsdtar.tarinfo.binary)
+    args.add(output)
+    args.add_all(ctx.files.controls)
+
+    ctx.actions.run(
+        executable = ctx.executable._dpkg_status_sh,
+        inputs = ctx.files.controls,
+        outputs = [output],
+        tools = bsdtar.default.files,
+        arguments = [args],
+    )
+
+    return [
+        DefaultInfo(files = depset([output])),
+    ]
+
+dpkg_status = rule(
+    doc = _DOC,
+    attrs = {
+        "_dpkg_status_sh": attr.label(
+            allow_single_file = True,
+            executable = True,
+            cfg = "exec",
+            default = ":dpkg_status.sh",
+        ),
+        "controls": attr.label_list(
+            allow_files = [".tar.xz", ".tar.gz", ".tar"],
+            mandatory = True,
+        ),
+    },
+    implementation = _dpkg_status_impl,
+    toolchains = [tar_lib.TOOLCHAIN_TYPE],
+)

apt/private/dpkg_status.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -o pipefail -o errexit -o nounset
+
+readonly bsdtar="$1"
+readonly out="$2"
+shift 2
+
+tmp_out=$(mktemp)
+
+while  (( $# > 0 )); do
+    control="$($bsdtar -xf "$1"  --to-stdout ./control)"
+    echo "$control" | head -n 1 >> $tmp_out
+    echo "Status: install ok installed" >> $tmp_out
+    echo "$control" | tail -n +2 >> $tmp_out
+    echo "" >> $tmp_out
+    shift
+done
+
+echo "#mtree
+./var/lib/dpkg/status type=file uid=0 gid=0 mode=0644 contents=$tmp_out
+" | "$bsdtar" $@ -cf "$out" @-
+

apt/private/dpkg_statusd.bzl

@@ -0,0 +1,54 @@
+"dpkg_statusd"
+
+# buildifier: disable=bzl-visibility
+load("//distroless/private:tar.bzl", "tar_lib")
+
+_DOC = """TODO: docs"""
+
+def _dpkg_statusd_impl(ctx):
+    bsdtar = ctx.toolchains[tar_lib.TOOLCHAIN_TYPE]
+
+    ext = tar_lib.common.compression_to_extension[ctx.attr.compression] if ctx.attr.compression else ".tar"
+    output = ctx.actions.declare_file(ctx.attr.name + ext)
+
+    args = ctx.actions.args()
+    args.add(bsdtar.tarinfo.binary)
+    args.add(output)
+    args.add(ctx.file.control)
+    args.add(ctx.attr.package_name)
+    tar_lib.common.add_compression_args(ctx.attr.compression, args)
+
+    ctx.actions.run(
+        executable = ctx.executable._dpkg_statusd_sh,
+        inputs = [ctx.file.control],
+        outputs = [output],
+        tools = bsdtar.default.files,
+        arguments = [args],
+    )
+
+    return [
+        DefaultInfo(files = depset([output])),
+    ]
+
+dpkg_statusd = rule(
+    doc = _DOC,
+    attrs = {
+        "_dpkg_statusd_sh": attr.label(
+            allow_single_file = True,
+            executable = True,
+            cfg = "exec",
+            default = ":dpkg_statusd.sh",
+        ),
+        "package_name": attr.string(mandatory = True),
+        "control": attr.label(
+            allow_single_file = [".tar.xz", ".tar.gz", ".tar"],
+            mandatory = True,
+        ),
+        "compression": attr.string(
+            doc = "Compress the archive file with a supported algorithm.",
+            values = tar_lib.common.accepted_compression_types,
+        ),
+    },
+    implementation = _dpkg_statusd_impl,
+    toolchains = [tar_lib.TOOLCHAIN_TYPE],
+)

apt/private/dpkg_statusd.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -o pipefail -o errexit -o nounset
+
+readonly bsdtar="$1"
+readonly out="$2"
+readonly control_path="$3"
+readonly package_name="$4"
+shift 4
+
+include=(--include "^./control$" --include "^./md5sums$")
+
+tmp=$(mktemp -d)
+"$bsdtar" -xf "$control_path" "${include[@]}" -C "$tmp"
+
+"$bsdtar" -cf - $@ --format=mtree "${include[@]}" --options '!gname,!uname,!sha1,!nlink,!time' "@$control_path" | \
+awk -v pkg="$package_name" '{ 
+    if ($1=="#mtree") {
+        print $1; next
+    };  
+    sub(/^\.?\//, "", $1);
+    $1 = "./var/lib/dpkg/status.d/" pkg "/" $1 " contents=./" $1; 
+    print $0
+}'  | "$bsdtar" $@ -cf "$out" -C "$tmp/" @-