From e9a1be599ea3bb875f20ade9f0b8237cbde59795 Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi Date: Fri, 5 Feb 2021 16:58:16 -0800 Subject: [PATCH 1/2] Nix based build shell for OE SDK. oe-nix-shell provides a nix based build environment for OE SDK. The build environment is reproducible. That is, on any machine that supports nix package manager, the same versions of build tools will be installed. The environment variables are also the same. Irrespective of what else is installed in the system, the build environment provided by oe-nix-shell is the same across all machines. Though this PR tackles one of the core aspects of reproducible builds, namely that the build environment is reproducible, it does not address reproducible builds of the OE SDK. This is by design. Reproducible builds of OE SDK can be attempted once the use of Nix has been vetted out in the OE SDK's development and CI. Signed-off-by: Anand Krishnamoorthi --- nix/oe-nix-shell | 134 +++++++++++++++++++ nix/shell.nix | 123 +++++++++++++++++ test/CMakeLists.txt | 5 + test/nix/CMakeLists.txt | 17 +++ test/nix/Testing/Temporary/CTestCostData.txt | 1 + test/nix/Testing/Temporary/LastTest.log | 3 + 6 files changed, 283 insertions(+) create mode 100755 nix/oe-nix-shell create mode 100755 nix/shell.nix create mode 100644 test/nix/CMakeLists.txt create mode 100644 test/nix/Testing/Temporary/CTestCostData.txt create mode 100644 test/nix/Testing/Temporary/LastTest.log diff --git a/nix/oe-nix-shell b/nix/oe-nix-shell new file mode 100755 index 0000000..cfe954e --- /dev/null +++ b/nix/oe-nix-shell @@ -0,0 +1,134 @@ +#!/bin/bash + +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +USAGE=" +Usage: oe-nix-shell [command] + oe-nix-shell -help + +To enter into OE SDK build shell, type oe-nix-shell +without any arguments. It will bring up a prompt: + [nix-shell:pwd]$ + +This is a nix-shell that has all the tools necessary +for building the OE SDK. To quit the shell, type exit. + +To execute a command and immediately exit the shell, +provide the command in quotes. + oe-nix-shell \"cd build && make -j 16\" + +This will enter the shell, change to the build folder, +build the SDK and exit the shell. +" + +# Print usage if wrong arguments supplied. +if [ "$#" -gt 1 ]; then + echo "${USAGE}" + exit 1 +fi + +# Print usage if requested. +if [ "$1" == "-help" ]; then + echo "${USAGE}" + exit 0 +fi + +# If no command is specified, use return; +# as the command to prevent nix-shell from +# exiting. +CMD="$1" +if [ "$#" -eq 0 ]; then + CMD="return;" +fi + +# Fetch the directory of oe-nix-shell script. +OE_NIX_SHELL_SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") + +# Command to enter into OE SDK nix shell +OE_NIX_SHELL_CMD="nix-shell \"${OE_NIX_SHELL_SCRIPT_DIR}/shell.nix\" --pure \ + --command \"neofetch; \ + echo Welcome to OE SDK build shell; \ + ${CMD}\"" + +# If nix is available in path, use it. +if command -v nix-shell &> /dev/null +then + bash -c "${OE_NIX_SHELL_CMD}" + exit 0 +fi + + +# Path to profile in single user mode install of nix. +NIX_PROFILE=~/.nix-profile/etc/profile.d/nix.sh + +# If root user, then install nix as per instructions here: +# https://github.com/NixOS/nix/issues/936 +if [ "$EUID" -eq 0 ]; then + + # Set user so that nix profile gets loaded correctly. + export USER=root + + # If a profile exists, the load it. + if test -f "${NIX_PROFILE}"; then + source "${NIX_PROFILE}" + fi + + #If nix-shell exists, use it. + if command -v nix-shell > /dev/null; then + bash -c "${OE_NIX_SHELL_CMD}" + exit 0 + fi + + # Install nix for root user. + # First create /nix folder + mkdir -p -m 0755 /nix + + # Create empty build-users-group + mkdir -p /etc/nix + echo "build-users-group =" > /etc/nix/nix.conf + + # Install nix. + # Assume debian based distro for now. + apt-get install -y curl + wget https://nixos.org/nix/install -O install.sh + curl -L https://nixos.org/nix/install | sh + + bash -c "source ${NIX_PROFILE} && ${OE_NIX_SHELL_CMD}" + exit 0 +fi + + +# Path to nix-user-chroot binary. +NIX_USER_CHROOT=~/.oe-nix/nix-user-chroot + +# Path to nix directory +NIX_DIR=~/.oe-nix/.nix + +# Ensure that OE nix directory exists. +mkdir -m 0755 -p ${NIX_DIR} +# Download nix-user-chroot +if ! test -f "${NIX_USER_CHROOT}" +then + echo "Downloading nix-user-chroot" + mkdir -p ~/.oe-nix + wget https://github.com/nix-community/nix-user-chroot/releases/download/1.1.1/nix-user-chroot-bin-1.1.1-x86_64-unknown-linux-musl \ + -O "${NIX_USER_CHROOT}" + chmod +x "${NIX_USER_CHROOT}" +fi + + +# Check if nix has been installed in the user chroot. +if ! "${NIX_USER_CHROOT}" "${NIX_DIR}" \ + bash -c "source ${NIX_PROFILE} && \ + which nix-shell > /dev/null" +then + # Install nix in the user chroot. + "${NIX_USER_CHROOT}" "${NIX_DIR}" \ + bash -c "sh <(curl -L https://nixos.org/nix/install) --no-daemon" +fi + +# Enter the shell. +"${NIX_USER_CHROOT}" "${NIX_DIR}" \ + bash -c "source ${NIX_PROFILE} && ${OE_NIX_SHELL_CMD}" + diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100755 index 0000000..273686f --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,123 @@ +let + # Let's choose the most recent nixpkgs release for installing + # most packages. The URL for nixpkgs tarball can be found from + # will be installed. The URL for the channel can be found from + # https://github.com/NixOS/nixpkgs/releases + # Nix has a 6 month release cycle and the releases are typically + # numbered year.03 and year.09. + # Using tarballs is *much* faster than using git to clone the + # nixpkgs repo. Additionally, tarballs pin the packages to + # specific versions. + # To obtain the sha256 for a tarball do: + # nix-prefetch-url --unpack + # That will fetch the tarball, unpack it and print the sha256. + pkgs = import (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/20.09.tar.gz"; + sha256 = "1wg61h4gndm3vcprdcg7rc4s1v3jkm5xd7lw8r2f67w502y94gcy"; + }) {}; + + # Some build tools like clang exist as different packages e.g + # clang_8, clang_9 etc in the same nixpkgs release. So it is + # trivial to choose the appropriate one: pkgs.clang_8 or pkgs.clang_9. + # Other packages like clang-format-7 may not be available in the latest + # nixpkg release. To find out how to install an older version of + # package, choose the channel and the specify the package name here: + # https://lazamar.co.uk/nix-versions/ + # That will bring up a list of older versions of the package. + # Clicking on the version we want will provide the nix code for + # installing that specific package. + # For example, the following lists older revisions of clang-tools: + # https://lazamar.co.uk/nix-versions/?channel=nixos-20.03&package=clang-tools + # The instructions for version 7.0.1 provide us the path to the tarball. + olderPkgs = import (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/7ff8a16f0726342f0a25697867d8c1306d4da7b0.tar.gz"; + sha256 = "0gwwwzriq4vv0v468mrx17lrmzr34lcdccmsxdfmw7z85fm8afdl"; + }) {}; +in + # Create development environment for building OE SDK. + # stdenvNoCC means that GCC (default) ought not be made available. + pkgs.stdenvNoCC.mkDerivation ({ + name = "oe-build-env"; + buildInputs = [ + # It is good to have git within the shell + # to work with other repos. + # version=2.29.2 + pkgs.git + + # version=3.18.2 + pkgs.cmake + + # Order of package declaration matters. + # Specify clang-tools first so that the + # older clang-format (7.0.1) appears before the + # newer clang-format (8.0.1) that is part of clang_8. + # version=7.0.1 + olderPkgs.clang-tools + + # OE SDK is built using clang. + # version=8.0.1 + pkgs.clang_8 + + # Use ccache to speed up recompilation. + # See shellHooks for configuration. + pkgs.ccache + + # Allow using ninja instead of make. + # version=1û10.1 + pkgs.ninja + + # OpenSSL tests require. Perl. + # The specific version may not matter. + # version=5.32.0 + pkgs.perl + + # Python is used in many places in the SDK. + # See shellHooks for how cmake-format is setup. + # version=3.8.5 + pkgs.python3 + + # Recent version of openssl is used. + # This may need to be selectively updated as newer + # versions are released. + # version=1.1.1g + pkgs.openssl + + # Doxygen for SDK document generation. + # version=1.8.19 + pkgs.doxygen + + # Graphviz is needed for the dot commant used by the + # document generation. + # dot version=2.43.0 + pkgs.graphviz + + # Needed for debugging the SDK. + # version=9.2 + pkgs.gdb + + # Fun + pkgs.neofetch + ]; + shellHooks = '' + # Set up C and C++ compilers explicitly via + # environment variables. This ensures that + # these compilers are picked up even if the + # shell has some other compilers lying around. + export CC=clang + export CXX=clang++ + + # Set up the base dir so that ccache uses relative + # paths from the base dir. This will result in + # cache hits across many local repos. + export CCACHE_BASEDIR=`pwd` + + # Setup python virtual env and install cmake-format. + # There might be better ways to install cmake-format in + # nix, but this is good enough for now. + rm -rf .venv + python -m venv .venv + source .venv/bin/activate + # Send "upgrade pip" warning to /dev/null. + pip install cmake-format==0.6.9 2>/dev/null + ''; + }) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 411d131..2422547 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -8,6 +8,11 @@ add_subdirectory(call_conflict) add_subdirectory(cmdline) add_subdirectory(comprehensive) add_subdirectory(import) + +if (UNIX) + add_subdirectory(nix) +endif () + add_subdirectory(prefix) add_subdirectory(preprocessor) add_subdirectory(safe_math) diff --git a/test/nix/CMakeLists.txt b/test/nix/CMakeLists.txt new file mode 100644 index 0000000..5670e28 --- /dev/null +++ b/test/nix/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) Open Enclave SDK contributors. +# Licensed under the MIT License. + +set(PASS_STRING "oe-nix-shell build completed") + +add_test( + NAME oe-nix-shell-test + COMMAND + ${PROJECT_SOURCE_DIR}/nix/oe-nix-shell "rm -rf build && \ + mkdir build && \ + cd build && \ + cmake ${PROJECT_SOURCE_DIR} && \ + make -j 8 && \ + echo ${PASS_STRING}") + +set_tests_properties(oe-nix-shell-test PROPERTIES PASS_REGULAR_EXPRESSION + "${PASS_STRING}") diff --git a/test/nix/Testing/Temporary/CTestCostData.txt b/test/nix/Testing/Temporary/CTestCostData.txt new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/test/nix/Testing/Temporary/CTestCostData.txt @@ -0,0 +1 @@ +--- diff --git a/test/nix/Testing/Temporary/LastTest.log b/test/nix/Testing/Temporary/LastTest.log new file mode 100644 index 0000000..1d7a362 --- /dev/null +++ b/test/nix/Testing/Temporary/LastTest.log @@ -0,0 +1,3 @@ +Start testing: Feb 05 14:32 PST +---------------------------------------------------------- +End testing: Feb 05 14:32 PST From 919092f14cc01347d0f0d6daecf3c90c3f050698 Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi Date: Mon, 8 Feb 2021 11:25:13 -0800 Subject: [PATCH 2/2] Intentionally fail the test Signed-off-by: Anand Krishnamoorthi --- test/nix/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nix/CMakeLists.txt b/test/nix/CMakeLists.txt index 5670e28..98aa865 100644 --- a/test/nix/CMakeLists.txt +++ b/test/nix/CMakeLists.txt @@ -14,4 +14,4 @@ add_test( echo ${PASS_STRING}") set_tests_properties(oe-nix-shell-test PROPERTIES PASS_REGULAR_EXPRESSION - "${PASS_STRING}") + "Intentionally fail the test to look at the test logs")