diff --git a/default.nix b/default.nix index a2db41493..68e4aaf7f 100644 --- a/default.nix +++ b/default.nix @@ -1,9 +1,9 @@ -{ system ? null, allInOne ? true, incremental ? false, img_tag ? "", tag ? "", img_org ? "" }: +{ system ? null, allInOne ? true, incremental ? false, static ? false, img_tag ? "", tag ? "", img_org ? "" }: let sources = import ./nix/sources.nix; hostSystem = (import sources.nixpkgs { }).hostPlatform.system; pkgs = import sources.nixpkgs { - overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { inherit allInOne incremental img_tag tag img_org; }) (import sources.rust-overlay) ]; + overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { inherit allInOne incremental static img_tag tag img_org; }) (import sources.rust-overlay) ]; system = if system != null then system else hostSystem; }; in diff --git a/nix/overlay.nix b/nix/overlay.nix index 365af694e..f75be9084 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,8 +1,8 @@ -{ allInOne ? true, incremental ? false, img_tag ? "", tag ? "", img_org ? "" }: +{ allInOne ? true, incremental ? false, static ? false, img_tag ? "", tag ? "", img_org ? "" }: self: super: { sourcer = super.callPackage ./lib/sourcer.nix { }; images = super.callPackage ./pkgs/images { inherit img_tag img_org; }; - extensions = super.callPackage ./pkgs/extensions { inherit allInOne incremental tag; }; + extensions = super.callPackage ./pkgs/extensions { inherit allInOne incremental static tag; }; openapi-generator = super.callPackage ./../dependencies/control-plane/nix/pkgs/openapi-generator { }; utils = super.callPackage ./pkgs/utils { inherit incremental; }; channel = import ./lib/rust.nix { pkgs = super.pkgs; }; diff --git a/nix/pkgs/extensions/cargo-project.nix b/nix/pkgs/extensions/cargo-project.nix index 8bca4fd7a..c01389dfd 100644 --- a/nix/pkgs/extensions/cargo-project.nix +++ b/nix/pkgs/extensions/cargo-project.nix @@ -26,16 +26,28 @@ # for this we use naersk which is not as fully fledged as the builtin rustPlatform so it should only be used # for development and not for CI , incremental ? false +, static ? false }: let stable_channel = { rustc = channel.default.stable; cargo = channel.default.stable; }; - rustPlatform = makeRustPlatform { - rustc = stable_channel.rustc; - cargo = stable_channel.cargo; + static-target = pkgs.rust.toRustTargetSpec pkgs.pkgsStatic.hostPlatform; + static-channel = channel.rust_default { + override = { targets = [ "${static-target}" ]; }; }; + rustPlatform = + if static then + pkgs.pkgsStatic.makeRustPlatform + { + rustc = static-channel.stable; + cargo = static-channel.stable; + } else + makeRustPlatform { + rustc = stable_channel.rustc; + cargo = stable_channel.cargo; + }; naersk = pkgs.callPackage sources.naersk { rustc = stable_channel.rustc; cargo = stable_channel.cargo; @@ -64,6 +76,10 @@ let "k8s" ]; src = sourcer.whitelistSource ../../../. src_list; + static_ssl = (pkgs.pkgsStatic.openssl.override { + static = true; + }); + hostTarget = pkgs.rust.toRustTargetSpec pkgs.hostPlatform; buildProps = rec { name = "extensions-${version}"; inherit version src; @@ -94,18 +110,33 @@ let build_with_default = { buildType, cargoBuildFlags }: rustPlatform.buildRustPackage (buildProps // { inherit buildType cargoBuildFlags; - preBuild = "patchShebangs ./dependencies/control-plane/scripts/rust/"; + preBuild = '' + patchShebangs ./dependencies/control-plane/scripts/rust/ + '' + pkgs.lib.optionalString (static) '' + # the rust builder from nixpkgks does not parse target and just uses the host target... + export NIX_CC_WRAPPER_TARGET_HOST_${builtins.replaceStrings [ "-" ] [ "_" ] hostTarget}= + export OPENSSL_STATIC=1 + export OPENSSL_LIB_DIR=${static_ssl.out}/lib + export OPENSSL_INCLUDE_DIR=${static_ssl.dev}/include + ''; + ${if static then "RUSTFLAGS" else null} = [ "-C" "target-feature=+crt-static" ]; cargoLock = { lockFile = ../../../Cargo.lock; }; }); - builder = if incremental then build_with_naersk else build_with_default; + cargoDeps = rustPlatform.importCargoLock { + lockFile = ../../../Cargo.lock; + }; + builder = + if static then build_with_default + else if incremental then build_with_naersk else build_with_default; + buildAllInOne = if static then false else allInOne; in { - inherit PROTOC PROTOC_INCLUDE version src; + inherit PROTOC PROTOC_INCLUDE version src cargoDeps; build = { buildType, cargoBuildFlags ? [ ] }: - if allInOne then + if buildAllInOne then builder { inherit buildType; cargoBuildFlags = [ "-p rpc" "-p metrics-exporter" "-p call-home" "-p upgrade" ]; } else builder { inherit buildType cargoBuildFlags; }; diff --git a/nix/pkgs/extensions/default.nix b/nix/pkgs/extensions/default.nix index 6eb744f5e..808de4753 100644 --- a/nix/pkgs/extensions/default.nix +++ b/nix/pkgs/extensions/default.nix @@ -1,4 +1,4 @@ -{ stdenv, git, lib, pkgs, allInOne, incremental, sourcer, tag ? "" }: +{ stdenv, git, lib, pkgs, allInOne, incremental, static, sourcer, tag ? "" }: let versionDrv = import ../../lib/version.nix { inherit sourcer lib stdenv git tag; }; version = builtins.readFile "${versionDrv}"; @@ -8,7 +8,7 @@ let "tag_or_long" = builtins.readFile "${versionDrv.tag_or_long}"; }; project-builder = - pkgs.callPackage ../extensions/cargo-project.nix { inherit sourcer gitVersions allInOne incremental; }; + pkgs.callPackage ../extensions/cargo-project.nix { inherit sourcer gitVersions allInOne incremental static; }; installer = { pname, src, suffix ? "" }: stdenv.mkDerivation rec { inherit pname src; @@ -70,6 +70,13 @@ let pname = "obs-callhome-stats"; }; }; + kubectl-plugin = installer { + src = builder.build { + inherit buildType; + cargoBuildFlags = [ "--bin kubectl-mayastor" ]; + }; + pname = "kubectl-mayastor"; + }; }; in { diff --git a/scripts/release.sh b/scripts/release.sh index fb767d257..2600f043e 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -30,6 +30,39 @@ nix_experimental() { echo -n " " fi } +pre_fetch_cargo_deps() { + local nixAttrPath=$1 + local project=$2 + local maxAttempt=$3 + + local outLink="--no-out-link" + local cargoVendorMsg="" + if [ -n "$CARGO_VENDOR_DIR" ]; then + if [ "$(realpath -s "$CARGO_VENDOR_DIR")" = "$(realpath -s "$SCRIPTDIR/..")" ]; then + cargoVendorDir="$CARGO_VENDOR_DIR/$GIT_BRANCH" + else + cargoVendorDir="$CARGO_VENDOR_DIR/$project/$GIT_BRANCH" + fi + + outLink="--out-link "$cargoVendorDir"" + cargoVendorMsg="into $(realpath -s "$cargoVendorDir") " + fi + + for (( attempt=1; attempt<=maxAttempt; attempt++ )); do + if $NIX_BUILD $outLink -A "$nixAttrPath"; then + echo "Cargo vendored dependencies pre-fetched "$cargoVendorMsg"after $attempt attempt(s)" + return 0 + fi + sleep 1 + done + if [ "$attempt" = "1" ]; then + echo "Cargo vendor pre-fetch is disabled" + return 0 + fi + + echo "Failed to pre-fetch the cargo vendored dependencies in $maxAttempt attempts" + exit 1 +} cleanup_and_exit() { local -r status=${1} @@ -81,8 +114,8 @@ RM="rm" SCRIPTDIR=$(dirname "$0") TAG=`get_tag` HASH=`get_hash` -BRANCH=`git rev-parse --abbrev-ref HEAD` -BRANCH=${BRANCH////-} +GIT_BRANCH=`git rev-parse --abbrev-ref HEAD` +BRANCH=${GIT_BRANCH////-} IMAGES= DEFAULT_IMAGES="metrics.exporter.io-engine obs.callhome stats.aggregator upgrade.job" IMAGES_THAT_REQUIRE_HELM_CHART=("upgrade.job") @@ -97,11 +130,12 @@ ALL_IN_ONE="true" INCREMENTAL="false" DEFAULT_BINARIES="kubectl-plugin" BUILD_BINARIES= -BIN_TARGET_PLAT="linux-musl" BINARY_OUT_LINK="." # This variable will be used to flag if the helm chart dependencies have been # been updated. HELM_DEPS_UPDATED="false" +CARGO_VENDOR_DIR=${CARGO_VENDOR_DIR:-} +CARGO_VENDOR_ATTEMPTS=${CARGO_VENDOR_ATTEMPTS:-25} # Check if all needed tools are installed curl --version >/dev/null @@ -199,6 +233,9 @@ trap 'cleanup_and_exit "$?"' EXIT cd $SCRIPTDIR/.. +# pre-fetch build dependencies with a number of attempts to harden against flaky networks +pre_fetch_cargo_deps extensions.project-builder.cargoDeps "mayastor-extensions" "$CARGO_VENDOR_ATTEMPTS" + if [ -z "$IMAGES" ]; then IMAGES="$DEFAULT_IMAGES" elif [ $(echo "$IMAGES" | wc -w) == "1" ]; then @@ -244,7 +281,7 @@ if [ -n "$BUILD_BINARIES" ]; then mkdir -p $BINARY_OUT_LINK for name in $BUILD_BINARIES; do echo "Building static $name ..." - $NIX_BUILD --out-link $BINARY_OUT_LINK/$name -A utils.$BUILD_TYPE.$BIN_TARGET_PLAT.$name + $NIX_BUILD --out-link $BINARY_OUT_LINK/$name -A extensions.$BUILD_TYPE.$name --arg static true done fi