chore(bors): merge pull request #484

484: build: harden build by pre-fetching dependencies r=tiagolobocastro a=tiagolobocastro Attempt to vendor the cargo deps for up to 25 times Supports linking the resulting derivation into a global location through env var CARGO_VENDOR_DIR, example: > CARGO_VENDOR_DIR=/tmp ./scripts/release.sh --image "" /nix/store/2x03mh7l1q0jx4xfsd3nw4h4ks0pxxya-cargo-vendor-dir Cargo vendored dependencies pre-fetched into /tmp/mayastor-controller/develop after 1 attempt(s) This can allow us to create a place holder in the jenkins nodes to keep the last dependencies for each main branch of each repo. <! Co-authored-by: Tiago Castro <[email protected]>
openebs · Apr 12, 2024 · fd34882 · fd34882
2 parents 6349ef2 + f63f4b3
commit fd34882
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 17 deletions.
diff --git a/default.nix b/default.nix
@@ -1,9 +1,9 @@
-{ system ? null, allInOne ? true, incremental ? false, img_tag ? "", tag ? "", img_org ? "" }:
+{ system ? null, allInOne ? true, incremental ? false, static ? false, img_tag ? "", tag ? "", img_org ? "" }:
 let
   sources = import ./nix/sources.nix;
   hostSystem = (import sources.nixpkgs { }).hostPlatform.system;
   pkgs = import sources.nixpkgs {
-    overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { inherit allInOne incremental img_tag tag img_org; }) (import sources.rust-overlay) ];
+    overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { inherit allInOne incremental static img_tag tag img_org; }) (import sources.rust-overlay) ];
     system = if system != null then system else hostSystem;
   };
 in

diff --git a/nix/overlay.nix b/nix/overlay.nix
@@ -1,8 +1,8 @@
-{ allInOne ? true, incremental ? false, img_tag ? "", tag ? "", img_org ? "" }:
+{ allInOne ? true, incremental ? false, static ? false, img_tag ? "", tag ? "", img_org ? "" }:
 self: super: {
   sourcer = super.callPackage ./lib/sourcer.nix { };
   images = super.callPackage ./pkgs/images { inherit img_tag img_org; };
-  extensions = super.callPackage ./pkgs/extensions { inherit allInOne incremental tag; };
+  extensions = super.callPackage ./pkgs/extensions { inherit allInOne incremental static tag; };
   openapi-generator = super.callPackage ./../dependencies/control-plane/nix/pkgs/openapi-generator { };
   utils = super.callPackage ./pkgs/utils { inherit incremental; };
   channel = import ./lib/rust.nix { pkgs = super.pkgs; };

diff --git a/nix/pkgs/extensions/cargo-project.nix b/nix/pkgs/extensions/cargo-project.nix
@@ -26,16 +26,28 @@
   # for this we use naersk which is not as fully fledged as the builtin rustPlatform so it should only be used
   # for development and not for CI
 , incremental ? false
+, static ? false
 }:
 let
   stable_channel = {
     rustc = channel.default.stable;
     cargo = channel.default.stable;
   };
-  rustPlatform = makeRustPlatform {
-    rustc = stable_channel.rustc;
-    cargo = stable_channel.cargo;
+  static-target = pkgs.rust.toRustTargetSpec pkgs.pkgsStatic.hostPlatform;
+  static-channel = channel.rust_default {
+    override = { targets = [ "${static-target}" ]; };
   };
+  rustPlatform =
+    if static then
+      pkgs.pkgsStatic.makeRustPlatform
+        {
+          rustc = static-channel.stable;
+          cargo = static-channel.stable;
+        } else
+      makeRustPlatform {
+        rustc = stable_channel.rustc;
+        cargo = stable_channel.cargo;
+      };
   naersk = pkgs.callPackage sources.naersk {
     rustc = stable_channel.rustc;
     cargo = stable_channel.cargo;
@@ -64,6 +76,10 @@ let
     "k8s"
   ];
   src = sourcer.whitelistSource ../../../. src_list;
+  static_ssl = (pkgs.pkgsStatic.openssl.override {
+    static = true;
+  });
+  hostTarget = pkgs.rust.toRustTargetSpec pkgs.hostPlatform;
   buildProps = rec {
     name = "extensions-${version}";
     inherit version src;
@@ -94,18 +110,33 @@ let
   build_with_default = { buildType, cargoBuildFlags }:
     rustPlatform.buildRustPackage (buildProps // {
       inherit buildType cargoBuildFlags;
-      preBuild = "patchShebangs ./dependencies/control-plane/scripts/rust/";
+      preBuild = ''
+        patchShebangs ./dependencies/control-plane/scripts/rust/
+      '' + pkgs.lib.optionalString (static) ''
+        # the rust builder from nixpkgks does not parse target and just uses the host target...
+        export NIX_CC_WRAPPER_TARGET_HOST_${builtins.replaceStrings [ "-" ] [ "_" ] hostTarget}=
+        export OPENSSL_STATIC=1
+        export OPENSSL_LIB_DIR=${static_ssl.out}/lib
+        export OPENSSL_INCLUDE_DIR=${static_ssl.dev}/include
+      '';
+      ${if static then "RUSTFLAGS" else null} = [ "-C" "target-feature=+crt-static" ];
       cargoLock = {
         lockFile = ../../../Cargo.lock;
       };
     });
-  builder = if incremental then build_with_naersk else build_with_default;
+  cargoDeps = rustPlatform.importCargoLock {
+    lockFile = ../../../Cargo.lock;
+  };
+  builder =
+    if static then build_with_default
+    else if incremental then build_with_naersk else build_with_default;
+  buildAllInOne = if static then false else allInOne;
 in
 {
-  inherit PROTOC PROTOC_INCLUDE version src;
+  inherit PROTOC PROTOC_INCLUDE version src cargoDeps;
 
   build = { buildType, cargoBuildFlags ? [ ] }:
-    if allInOne then
+    if buildAllInOne then
       builder { inherit buildType; cargoBuildFlags = [ "-p rpc" "-p metrics-exporter" "-p call-home" "-p upgrade" ]; }
     else
       builder { inherit buildType cargoBuildFlags; };

diff --git a/nix/pkgs/extensions/default.nix b/nix/pkgs/extensions/default.nix
@@ -1,4 +1,4 @@
-{ stdenv, git, lib, pkgs, allInOne, incremental, sourcer, tag ? "" }:
+{ stdenv, git, lib, pkgs, allInOne, incremental, static, sourcer, tag ? "" }:
 let
   versionDrv = import ../../lib/version.nix { inherit sourcer lib stdenv git tag; };
   version = builtins.readFile "${versionDrv}";
@@ -8,7 +8,7 @@ let
     "tag_or_long" = builtins.readFile "${versionDrv.tag_or_long}";
   };
   project-builder =
-    pkgs.callPackage ../extensions/cargo-project.nix { inherit sourcer gitVersions allInOne incremental; };
+    pkgs.callPackage ../extensions/cargo-project.nix { inherit sourcer gitVersions allInOne incremental static; };
   installer = { pname, src, suffix ? "" }:
     stdenv.mkDerivation rec {
       inherit pname src;
@@ -70,6 +70,13 @@ let
         pname = "obs-callhome-stats";
       };
     };
+    kubectl-plugin = installer {
+      src = builder.build {
+        inherit buildType;
+        cargoBuildFlags = [ "--bin kubectl-mayastor" ];
+      };
+      pname = "kubectl-mayastor";
+    };
   };
 in
 {

diff --git a/scripts/release.sh b/scripts/release.sh
@@ -30,6 +30,39 @@ nix_experimental() {
       echo -n " "
   fi
 }
+pre_fetch_cargo_deps() {
+  local nixAttrPath=$1
+  local project=$2
+  local maxAttempt=$3
+
+  local outLink="--no-out-link"
+  local cargoVendorMsg=""
+  if [ -n "$CARGO_VENDOR_DIR" ]; then
+    if [ "$(realpath -s "$CARGO_VENDOR_DIR")" = "$(realpath -s "$SCRIPTDIR/..")" ]; then
+      cargoVendorDir="$CARGO_VENDOR_DIR/$GIT_BRANCH"
+    else
+      cargoVendorDir="$CARGO_VENDOR_DIR/$project/$GIT_BRANCH"
+    fi
+
+    outLink="--out-link "$cargoVendorDir""
+    cargoVendorMsg="into $(realpath -s "$cargoVendorDir") "
+  fi
+
+  for (( attempt=1; attempt<=maxAttempt; attempt++ )); do
+     if $NIX_BUILD $outLink -A "$nixAttrPath"; then
+       echo "Cargo vendored dependencies pre-fetched "$cargoVendorMsg"after $attempt attempt(s)"
+       return 0
+     fi
+     sleep 1
+  done
+  if [ "$attempt" = "1" ]; then
+    echo "Cargo vendor pre-fetch is disabled"
+    return 0
+  fi
+
+  echo "Failed to pre-fetch the cargo vendored dependencies in $maxAttempt attempts"
+  exit 1
+}
 cleanup_and_exit() {
   local -r status=${1}
 
@@ -81,8 +114,8 @@ RM="rm"
 SCRIPTDIR=$(dirname "$0")
 TAG=`get_tag`
 HASH=`get_hash`
-BRANCH=`git rev-parse --abbrev-ref HEAD`
-BRANCH=${BRANCH////-}
+GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+BRANCH=${GIT_BRANCH////-}
 IMAGES=
 DEFAULT_IMAGES="metrics.exporter.io-engine obs.callhome stats.aggregator upgrade.job"
 IMAGES_THAT_REQUIRE_HELM_CHART=("upgrade.job")
@@ -97,11 +130,12 @@ ALL_IN_ONE="true"
 INCREMENTAL="false"
 DEFAULT_BINARIES="kubectl-plugin"
 BUILD_BINARIES=
-BIN_TARGET_PLAT="linux-musl"
 BINARY_OUT_LINK="."
 # This variable will be used to flag if the helm chart dependencies have been
 # been updated.
 HELM_DEPS_UPDATED="false"
+CARGO_VENDOR_DIR=${CARGO_VENDOR_DIR:-}
+CARGO_VENDOR_ATTEMPTS=${CARGO_VENDOR_ATTEMPTS:-25}
 
 # Check if all needed tools are installed
 curl --version >/dev/null
@@ -199,6 +233,9 @@ trap 'cleanup_and_exit "$?"' EXIT
 
 cd $SCRIPTDIR/..
 
+# pre-fetch build dependencies with a number of attempts to harden against flaky networks
+pre_fetch_cargo_deps extensions.project-builder.cargoDeps "mayastor-extensions" "$CARGO_VENDOR_ATTEMPTS"
+
 if [ -z "$IMAGES" ]; then
   IMAGES="$DEFAULT_IMAGES"
 elif [ $(echo "$IMAGES" | wc -w) == "1" ]; then
@@ -244,7 +281,7 @@ if [ -n "$BUILD_BINARIES" ]; then
   mkdir -p $BINARY_OUT_LINK
   for name in $BUILD_BINARIES; do
     echo "Building static $name ..."
-    $NIX_BUILD --out-link $BINARY_OUT_LINK/$name -A utils.$BUILD_TYPE.$BIN_TARGET_PLAT.$name
+    $NIX_BUILD --out-link $BINARY_OUT_LINK/$name -A extensions.$BUILD_TYPE.$name --arg static true
   done
 fi