Post review improvements

- save crates to the correct deps/cargo path. - add .cargo/config.toml as project file - format crates as a folder with sha's for compatibility with cargo vendor (and thus removing the need to use crates indexes)
containerbuildsystem · Oct 18, 2023 · 94ce383 · 94ce383
1 parent c1b3940
commit 94ce383
Show file tree

Hide file tree

Showing 2 changed files with 148 additions and 52 deletions.
diff --git a/cachi2/core/package_managers/cargo.py b/cachi2/core/package_managers/cargo.py
@@ -1,12 +1,16 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
+import hashlib
+import json
 import logging
+import tarfile
 from pathlib import Path
+from textwrap import dedent
 
 import tomli
 
 from cachi2.core.checksum import ChecksumInfo, must_match_any_checksum
 from cachi2.core.models.input import Request
-from cachi2.core.models.output import Component, RequestOutput
+from cachi2.core.models.output import Component, ProjectFile, RequestOutput
 from cachi2.core.package_managers.general import download_binary_file
 from cachi2.core.rooted_path import RootedPath
 
@@ -32,10 +36,23 @@ def fetch_cargo_source(request: Request) -> RequestOutput:
         for dependency in info["dependencies"]:
             components.append(Component.from_package_dict(dependency))
 
+    cargo_config = ProjectFile(
+        abspath=request.source_dir.join_within_root(".cargo/config.toml"),
+        template=dedent(
+            """
+            [source.crates-io]
+            replace-with = "local"
+
+            [source.local]
+            directory = "${output_dir}/deps/cargo"
+            """
+        ),
+    )
+
     return RequestOutput.from_obj_list(
         components=components,
         environment_variables=[],
-        project_files=[],
+        project_files=[cargo_config],
     )
 
 
@@ -50,8 +67,7 @@ def _resolve_cargo(
     assert pkg_name and pkg_version, "INVALID PACKAGE"
 
     dependencies = []
-    if not lock_file:
-        lock_file = app_path / DEFAULT_LOCK_FILE
+    lock_file = app_path / (lock_file or DEFAULT_LOCK_FILE)
 
     cargo_lock_dict = tomli.load(lock_file.open("rb"))
     for dependency in cargo_lock_dict["package"]:
@@ -80,19 +96,63 @@ def _download_cargo_dependencies(output_path: RootedPath, cargo_dependencies: li
         checksum_info = ChecksumInfo(algorithm="sha256", hexdigest=dep["checksum"])
         dep_name = dep["name"]
         dep_version = dep["version"]
-        download_path = Path(output_path.join_within_root(f"{dep_name}-{dep_version}.crate"))
-        download_path.parent.mkdir(exist_ok=True)
+        download_path = Path(
+            output_path.join_within_root(f"deps/cargo/{dep_name}-{dep_version}.crate")
+        )
+        download_path.parent.mkdir(exist_ok=True, parents=True)
         download_url = f"https://crates.io/api/v1/crates/{dep_name}/{dep_version}/download"
         download_binary_file(download_url, download_path)
         must_match_any_checksum(download_path, [checksum_info])
+        vendored_dep = prepare_crate_as_vendored_dep(download_path)
         downloads.append(
             {
                 "package": dep_name,
                 "name": dep_name,
                 "version": dep_version,
-                "path": download_path,
+                "path": vendored_dep,
                 "type": "cargo",
                 "dev": False,
             }
         )
     return downloads
+
+def _calc_sha256(content: bytes):
+    return hashlib.sha256(content).hexdigest()
+
+def generate_cargo_checksum(crate_path: Path):
+    """Generate Cargo checksums
+
+    cargo requires vendored dependencies to have a ".cargo_checksum.json" BUT crates
+    downloaded from crates.io don't come with this file. This function generates
+    a dictionary compatible what cargo expects.
+
+    Args:
+        crate_path (Path): crate tarball
+
+    Returns:
+        dict: checksums expected by cargo
+    """
+    checksums = {"package": _calc_sha256(crate_path.read_bytes()), "files": {}}
+    tarball = tarfile.open(crate_path)
+    for tarmember in tarball.getmembers():
+        name = tarmember.name.split("/", 1)[1]  # ignore folder name
+        checksums["files"][name] = _calc_sha256(tarball.extractfile(tarmember.name).read())
+    tarball.close()
+    return checksums
+
+
+def prepare_crate_as_vendored_dep(crate_path: Path) -> Path:
+    """Prepare crates as vendored dependencies
+
+    Extracts contents from crate and add a ".cargo_checksum.json" file to it
+
+    Args:
+        crate_path (Path): crate tarball
+    """
+    checksums = generate_cargo_checksum(crate_path)
+    with tarfile.open(crate_path) as tarball:
+        folder_name = tarball.getnames()[0].split("/")[0]
+        tarball.extractall(crate_path.parent)
+    cargo_checksum = crate_path.parent / folder_name / ".cargo-checksum.json"
+    json.dump(checksums, cargo_checksum.open("w"))
+    return crate_path.parent / folder_name
diff --git a/docs/usage.md b/docs/usage.md
@@ -504,97 +504,133 @@ podman build . \
 ### Prefetch dependencies (cargo)
 
 ```shell
-mkdir -p playground/pure-rust
-cd playground/pure-rust
+mkdir -p /tmp/playground/pure-rust
+cd /tmp/playground/pure-rust
 git clone [email protected]:sharkdp/bat.git --branch=v0.22.1
 cachi2 fetch-deps --source bat '{"type": "cargo"}'
 ```
 
 ### Generate environment variables (cargo)
 
-#### Alternative crates.io
-Cargo must be configured differently when crates.io will be replaced with an alternative registry to crates.io (or at least similar way - something that the [cargo plugin for nexus](https://github.com/sonatype-nexus-community/nexus-repository-cargo) seems to handle) or from a local path.
+At the moment no env var is generated for cargo, but let's do this step for compatibility
+with other integrations.
 
-For enabling an alternative crates.io registry the only required env is 
-
-```
-CARGO_REGISTRIES_MY_REGISTRY_INDEX=https://my-intranet:8080/git/index
+```shell
+cachi2 generate-env ./cachi2-output -o ./cachi2.env --for-output-dir /tmp/cachi2-output
 ```
 
-[cargo docs on using an alternate registry](https://doc.rust-lang.org/cargo/reference/registries.html#using-an-alternate-registry)
+### Inject project files (cargo)
 
-Unfortunately I'm still clueless on how to run an alternative registry (and this don't seem to align with cachi2 ethos), so I'm skipping this option.
+```shell
+$ cachi2 inject-files $(realpath cachi2-output) --for-output-dir /tmp/cachi2-output
+2023-10-18 14:51:01,936 INFO Creating /tmp/playground/pure-rust/bat/.cargo/config.toml
+```
 
-#### Replacing crates.io with a local path
+### Build the base image (cargo)
 
-TLDR: not possible through environment variables; will need to inject a config file to a specific location
 
-Unfortunately this method does not work with environment variables (see [this cargo issue](https://github.com/rust-lang/cargo/issues/5416)) and requires a .cargo/config.toml file.
+Containerfile.baseimage
+```Dockerfile
+FROM registry.access.redhat.com/ubi9/ubi
 
-It must be placed somewhere relative to where "cargo install/build" will run following the hierarchical structure below
+RUN dnf install cargo rust rust-std-static -y &&\
+    dnf clean all
+```
 
->Cargo allows local configuration for a particular package as well as global configuration. It looks for configuration files in the current directory and all parent directories. If, for example, Cargo were invoked in `/projects/foo/bar/baz`, then the following configuration files would be probed for and unified in this order:
->-   `/projects/foo/bar/baz/.cargo/config.toml`
->-   `/projects/foo/bar/.cargo/config.toml`
->-   `/projects/foo/.cargo/config.toml`
->-   `/projects/.cargo/config.toml`
->-   `/.cargo/config.toml`
->-   `$CARGO_HOME/config.toml` which defaults to:
- >   -   Windows: `%USERPROFILE%\.cargo\config.toml`
->    -   Unix: `$HOME/.cargo/config.toml`
+```shell
+podman build --tag bat-base-image -f Containerfile.baseimage .
+```
 
-Source: [cargo docs](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure)
+### Build the application image (cargo)
 
-This is how I configured mine:
+Containerfile
+```Dockerfile
+FROM bat-base-image:latest
 
-cargo_config.toml
+COPY bat /app
+WORKDIR /app
+RUN source /tmp/cachi2.env && \
+    cargo install --locked --path .
+ENV PATH="/root/.cargo/bin:$PATH"
+CMD bat
 ```
-[source.crates-io]
-replace-with = "local"
 
-[source.local]
-local-registry = "/tmp/cachi2-output"
+```shell
+podman build . \
+  --volume "$(realpath ./cachi2-output)":/tmp/cachi2-output:Z \
+  --volume "$(realpath ./cachi2.env)":/tmp/cachi2.env:Z \
+  --network none \
+  --tag bat
 ```
-local-registry also requires a index of all packages downloaded exactly like what's in https://github.com/rust-lang/crates.io-index. For this POC I just git cloned this repo locally and placed it at `./cachi2-output/index` (NOTE: this is definitely overkill; the final version should use only indexes for the packages we have)
+
+## Example: pip with indirect cargo dependencies
+
+### Prefetch dependencies (pip + cargo)
 
 ```shell
-git clone [email protected]:rust-lang/crates.io-index.git --depth 1 cachi2-output/index
+mkdir -p /tmp/playground/python-cargo
+cd /tmp/playground/python-cargo
+git clone [email protected]:/bruno-fs/simple-python-rust-project --branch=0.0.1 dummy
+cachi2 fetch-deps --source dummy '[{"type": "pip"}, {"type": "cargo", "lock_file": "merged-cargo.lock", "pkg_name": "dummy", "pkg_version": "0.0.1"}]'
 ```
 
-There's a [cargo local registry package](https://crates.io/crates/cargo-local-registry) that prepares  dependencies like this, but it is pretty buggy and fails for some packages I played around (including the one used on this PoC).
+### Generate environment variables (pip + cargo)
 
-### Build the base image (cargo)
+```shell
+cachi2 generate-env ./cachi2-output -o ./cachi2.env --for-output-dir /tmp/cachi2-output
+```
+
+### Inject project files (pip + cargo)
+
+```shell
+$ cachi2 inject-files $(realpath cachi2-output) --for-output-dir /tmp/cachi2-output
+2023-10-18 14:51:01,936 INFO Creating /tmp/playground/python-cargo/dummy/.cargo/config.toml
+```
+
+### Build the base image (pip + cargo)
 
 
 Containerfile.baseimage
 ```Dockerfile
-FROM registry.redhat.io/ubi9/ubi
+FROM quay.io/centos/centos:stream8
 
-RUN dnf install cargo rust rust-std-static -y &&\
+RUN dnf -y install \
+        python3.11 \
+        python3.11-pip \
+        python3.11-devel \
+        gcc \
+        libffi-devel \
+        openssl-devel \
+        cargo \
+        rust \
+        rust-std-static &&\
     dnf clean all
 ```
 
 ```shell
-podman build --tag bat-base-image -f Containerfile.baseimage .
+podman build --tag dummy-base-image -f Containerfile.baseimage .
 ```
 
 ### Build the application image (cargo)
 
 Containerfile
 ```Dockerfile
-FROM bat-base-image:latest
+FROM dummy-base-image:latest
 
-COPY cargo_config.toml /app/.cargo/config.toml
-COPY bat /app
+COPY dummy /app
+# we don't have a way to control where pip will build
+# cargo dependencies, so we need to move cargo configuration
+# to the place where python run builds
+COPY dummy/.cargo/config.toml /tmp/.cargo/config.toml
 WORKDIR /app
-RUN cargo install --locked --path .
-ENV PATH="/root/.cargo/bin:$PATH"
-CMD bat
+RUN source /tmp/cachi2.env && \
+    pip3 install -r requirements.txt
 ```
 
 ```shell
 podman build . \
   --volume "$(realpath ./cachi2-output)":/tmp/cachi2-output:Z \
+  --volume "$(realpath ./cachi2.env)":/tmp/cachi2.env:Z \
   --network none \
-  --tag bat
-```
+  --tag dummy
+```