From 788d280db18862081d0551dabd5c198fc61aa287 Mon Sep 17 00:00:00 2001 From: Lukas Holzer Date: Mon, 17 Oct 2022 17:03:02 +0200 Subject: [PATCH] feat: add pnpm to build image (#845) * feat: add pnpm to build image * chore: add PR review * chore: always enable node corepack * chore: restore how we install yarn and check npm version for corepack * chore: cleanup * chore: revert yarn tests * feat: make yarn + corepack working together * Update run-build-functions.sh Co-authored-by: Nicola Aitken * chore: fix typo Co-authored-by: Nicola Aitken * chore: fix typo Co-authored-by: Nicola Aitken * chore: add PR feedback * chore: remove unused function Co-authored-by: Nicola Aitken --- .gitignore | 2 - Dockerfile | 14 +-- Makefile | 1 - focal.yaml | 3 +- run-build-functions.sh | 184 ++++++++++++++++++++++++-------------- run-build.sh | 3 +- test-tools/start-image.sh | 1 + test-tools/test-build.sh | 2 + tests/node/base.bats | 3 +- tests/node/pnpm.bats | 54 +++++++++++ tests/node/yarn.bats | 51 ++++++++--- 11 files changed, 229 insertions(+), 89 deletions(-) create mode 100644 tests/node/pnpm.bats diff --git a/.gitignore b/.gitignore index 228df3f9..2a20afff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ tmp/ node_modules -package-lock.json -yarn.lock diff --git a/Dockerfile b/Dockerfile index 55d445ba..d7ce222b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -274,29 +274,31 @@ USER root # ################################################################################ - +# this installer is needed for older node versions where no corepack is available RUN curl -o- -L https://yarnpkg.com/install.sh > /usr/local/bin/yarn-installer.sh ENV NVM_VERSION=0.39.1 -# Install node.js, yarn, grunt, bower and elm +# Install node.js, yarn, grunt, bower USER buildbot RUN git clone https://github.com/creationix/nvm.git ~/.nvm && \ cd ~/.nvm && \ git checkout v$NVM_VERSION && \ cd / -ENV ELM_VERSION=0.19.1-5 ENV YARN_VERSION=1.22.19 +ENV PNPM_VERSION=7.13.4 ENV NETLIFY_NODE_VERSION="16" RUN /bin/bash -c ". ~/.nvm/nvm.sh && \ nvm install --no-progress $NETLIFY_NODE_VERSION && \ npm install -g grunt-cli bower && \ - bash /usr/local/bin/yarn-installer.sh --version $YARN_VERSION && \ - nvm alias default node && nvm cache clear" -ENV PATH "/opt/buildhome/.yarn/bin:$PATH" + nvm alias default node && \ + nvm cache clear && \ + corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate && \ + corepack prepare pnpm@$PNPM_VERSION --activate" USER root diff --git a/Makefile b/Makefile index 4a30e74b..8e46246f 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,6 @@ container-test: build-base ## Run a container structure test run-local: build-base ## Volume the build scripts and run a bash shell in the build-image docker run --rm -it \ -e NETLIFY_BUILD_BASE="/opt/buildhome" \ - -u root \ -v $(PWD)/run-build.sh:/opt/build-bin/build:ro \ -v $(PWD)/run-build-functions.sh:/opt/build-bin/run-build-functions.sh:ro \ $(image) diff --git a/focal.yaml b/focal.yaml index 8536cf59..0016e076 100644 --- a/focal.yaml +++ b/focal.yaml @@ -30,7 +30,6 @@ metadataTest: /opt/buildhome/.php:\ /opt/buildhome/.binrc/bin:\ /opt/buildhome/.deno/bin:\ - /opt/buildhome/.yarn/bin:\ /usr/local/rvm/bin:\ /usr/local/sbin:\ /usr/local/bin:\ @@ -91,11 +90,13 @@ commandTests: echo "Node: $(node --version)" echo "Yarn: $(yarn --version)" echo "NPM: $(npm --version)" + echo "PNPM: $(pnpm --version)" expectedOutput: - "Node: v16.*.*" - "NVM: 0.39.1" - "Yarn: 1.22.*" - "NPM: 8.*.*" + - "PNPM: 7.*.*" - name: hugo command: "hugo" diff --git a/run-build-functions.sh b/run-build-functions.sh index 1d4ef2f4..52281de6 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -132,32 +132,38 @@ restore_node_modules() { run_yarn() { yarn_version=$1 - if [ -d $NETLIFY_CACHE_DIR/yarn ] - then - export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH - fi restore_home_cache ".yarn_cache" "yarn cache" - if [ $(which yarn) ] && [ "$(yarn --version)" != "$yarn_version" ] - then - echo "Found yarn version ($(yarn --version)) that doesn't match expected ($yarn_version)" - rm -rf $NETLIFY_CACHE_DIR/yarn $HOME/.yarn - npm uninstall yarn -g - fi + if ! [ $(which corepack) ]; then + if [ -d $NETLIFY_CACHE_DIR/yarn ] + then + export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH + fi - if ! [ $(which yarn) ] - then - echo "Installing yarn at version $yarn_version" - rm -rf $HOME/.yarn - bash /usr/local/bin/yarn-installer.sh --version $yarn_version - mv $HOME/.yarn $NETLIFY_CACHE_DIR/yarn - export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH + if [ $(which yarn) ] && [ "$(yarn --version)" != "$yarn_version" ]; then + echo "Found yarn version ($(yarn --version)) that doesn't match expected ($yarn_version)" + rm -rf $NETLIFY_CACHE_DIR/yarn $HOME/.yarn + npm uninstall yarn -g + fi + + if ! [ $(which yarn) ]; then + echo "Installing yarn at version $yarn_version" + rm -rf $HOME/.yarn + bash /usr/local/bin/yarn-installer.sh --version $yarn_version + mv $HOME/.yarn $NETLIFY_CACHE_DIR/yarn + export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH + fi + else + # if corepack is installed use it for changing the yarn version + if [ "$(yarn --version)" != "$yarn_version" ]; then + echo "Installing yarn at version $yarn_version" + corepack prepare yarn@$yarn_version --activate + fi fi restore_node_modules "yarn" echo "Installing NPM modules using Yarn version $(yarn --version)" - run_npm_set_temp # Remove the cache-folder flag if the user set any. # We want to control where to put the cache @@ -177,10 +183,38 @@ run_yarn() { export PATH=$(yarn bin):$PATH } -run_npm_set_temp() { - # Make sure we're not limited by space in the /tmp mount - mkdir $HOME/tmp - npm set tmp $HOME/tmp + +run_pnpm() { + pnpm_version=$1 + restore_home_cache ".pnpm-store" "pnpm cache" + + if ! [ $(which corepack) ]; then + echo "Error while installing PNPM $pnpm_version" + echo "We cannot install the expected version of PNPM ($pnpm_version) as your required Node.js version $NODE_VERSION does not allow that" + echo "Please ensure that you use at least Node Version 14.19.0 or greater than 16.9.0" + + exit 1 + fi + + if [ "$(pnpm --version)" != "$pnpm_version" ] + then + echo "Found pnpm version ($(pnpm --version)) that doesn't match expected ($pnpm_version)" + + corepack prepare pnpm@$pnpm_version --activate + fi + + restore_node_modules "pnpm" + + echo "Installing NPM modules using PNPM version $(pnpm --version)" + if pnpm install + then + echo "NPM modules installed using PNPM" + else + echo "Error during PNPM install" + exit 1 + fi + + export PATH=$(pnpm bin):$PATH } run_npm() { @@ -205,7 +239,7 @@ run_npm() { if install_deps package.json $NODE_VERSION $NETLIFY_CACHE_DIR/package-sha then echo "Installing NPM modules using NPM version $(npm --version)" - run_npm_set_temp + if npm install ${NPM_FLAGS:+$NPM_FLAGS} then echo "NPM modules installed" @@ -219,46 +253,9 @@ run_npm() { export PATH=$(npm bin):$PATH } -check_python_version() { - if source $HOME/python${PYTHON_VERSION}/bin/activate - then - echo "Python version set to ${PYTHON_VERSION}" - else - echo "Error setting python version from $1" - echo "Please see https://github.com/netlify/build-image/blob/focal/included_software.md for current versions" - exit 1 - fi -} +install_node() { + local defaultNodeVersion=$1 -read_node_version_file() { - local nodeVersionFile="$1" - NODE_VERSION="$(cat "$nodeVersionFile")" - echo "Attempting node version '$NODE_VERSION' from $nodeVersionFile" -} - -install_dependencies() { - local defaultNodeVersion=$1 # 16 - local defaultRubyVersion=$2 # 2.6.2 - local defaultYarnVersion=$3 # 1.13.0 - local installGoVersion=$4 # 1.16.4 - local defaultPythonVersion=$5 # 3.8 - local featureFlags="$6" - - # Python Version - if [ -f runtime.txt ] - then - PYTHON_VERSION=$(cat runtime.txt) - check_python_version "runtime.txt" - elif [ -f Pipfile ] - then - echo "Found Pipfile restoring Pipenv virtualenv" - restore_cwd_cache ".venv" "python virtualenv" - else - PYTHON_VERSION=$defaultPythonVersion - check_python_version "the PYTHON_VERSION environment variable" - fi - - # Node version source $NVM_DIR/nvm.sh : ${NODE_VERSION="$defaultNodeVersion"} @@ -296,6 +293,12 @@ install_dependencies() { exit 1 fi + # if Node.js Corepack is available enable it + if [ $(which corepack) ]; then + echo "Enabling node corepack" + corepack enable + fi + if [ -n "$NPM_TOKEN" ] then if [ ! -f .npmrc ] @@ -303,6 +306,50 @@ install_dependencies() { echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc fi fi +} + +check_python_version() { + if source $HOME/python${PYTHON_VERSION}/bin/activate + then + echo "Python version set to ${PYTHON_VERSION}" + else + echo "Error setting python version from $1" + echo "Please see https://github.com/netlify/build-image/blob/focal/included_software.md for current versions" + exit 1 + fi +} + +read_node_version_file() { + local nodeVersionFile="$1" + NODE_VERSION="$(cat "$nodeVersionFile")" + echo "Attempting node version '$NODE_VERSION' from $nodeVersionFile" +} + +install_dependencies() { + local defaultNodeVersion=$1 # 16 + local defaultRubyVersion=$2 # 2.6.2 + local defaultYarnVersion=$3 # 1.13.0 + local defaultPnpmVersion=$4 # 7.13.4 + local installGoVersion=$5 # 1.16.4 + local defaultPythonVersion=$6 # 3.8 + local featureFlags="$7" + + # Python Version + if [ -f runtime.txt ] + then + PYTHON_VERSION=$(cat runtime.txt) + check_python_version "runtime.txt" + elif [ -f Pipfile ] + then + echo "Found Pipfile restoring Pipenv virtualenv" + restore_cwd_cache ".venv" "python virtualenv" + else + PYTHON_VERSION=$defaultPythonVersion + check_python_version "the PYTHON_VERSION environment variable" + fi + + # Node version + install_node $defaultNodeVersion # Automatically installed Build plugins if [ ! -d "$PWD/.netlify" ] @@ -527,19 +574,22 @@ install_dependencies() { # NPM Dependencies : ${YARN_VERSION="$defaultYarnVersion"} + : ${PNPM_VERSION="$defaultPnpmVersion"} : ${CYPRESS_CACHE_FOLDER="./node_modules/.cache/CypressBinary"} export CYPRESS_CACHE_FOLDER if [ -f package.json ] then - if [ "$NODE_ENV" == "production" ] - then + if [ "$NODE_ENV" == "production" ];then warn "The environment variable 'NODE_ENV' is set to 'production'. Any 'devDependencies' in package.json will not be installed" fi - if [ "$NETLIFY_USE_YARN" = "true" ] || ([ "$NETLIFY_USE_YARN" != "false" ] && [ -f yarn.lock ]) - then + restore_home_cache ".node/corepack" "corepack dependencies" + + if [ "$NETLIFY_USE_YARN" = "true" ] || ([ "$NETLIFY_USE_YARN" != "false" ] && [ -f yarn.lock ]); then run_yarn $YARN_VERSION + elif [ "$NETLIFY_USE_PNPM" = "true" ] || ([ "$NETLIFY_USE_PNPM" != "false" ] && [ -f pnpm-lock.yaml ]); then + run_pnpm $PNPM_VERSION else run_npm "$featureFlags" fi @@ -707,6 +757,8 @@ cache_artifacts() { cache_cwd_directory_fast_copy "target" "rust compile output" fi + cache_home_directory ".node/corepack" "corepack cache" + cache_home_directory ".pnpm-store" "pnpm cache" cache_home_directory ".yarn_cache" "yarn cache" cache_home_directory ".cache/pip" "pip cache" cache_home_directory ".cask" "emacs cask dependencies" diff --git a/run-build.sh b/run-build.sh index 8ddfa657..9f4655a7 100755 --- a/run-build.sh +++ b/run-build.sh @@ -21,11 +21,12 @@ cd $NETLIFY_REPO_DIR : ${NODE_VERSION="16"} : ${RUBY_VERSION="2.7.2"} : ${YARN_VERSION="1.22.19"} +: ${PNPM_VERSION="7.13.4"} : ${GO_VERSION="1.17"} : ${PYTHON_VERSION="3.8"} echo "Installing dependencies" -install_dependencies $NODE_VERSION $RUBY_VERSION $YARN_VERSION $GO_VERSION $PYTHON_VERSION +install_dependencies $NODE_VERSION $RUBY_VERSION $YARN_VERSION $PNPM_VERSION $GO_VERSION $PYTHON_VERSION echo "Installing missing commands" install_missing_commands diff --git a/test-tools/start-image.sh b/test-tools/start-image.sh index 75fec7e9..e0290e54 100755 --- a/test-tools/start-image.sh +++ b/test-tools/start-image.sh @@ -9,6 +9,7 @@ docker run --rm -t -i \ -e NPM_VERSION \ -e RUBY_VERSION \ -e YARN_VERSION \ + -e PNPM_VERSION \ -e HUGO_VERSION \ -e PHP_VERSION \ -e GO_VERSION \ diff --git a/test-tools/test-build.sh b/test-tools/test-build.sh index 3f2e423c..9f48d930 100755 --- a/test-tools/test-build.sh +++ b/test-tools/test-build.sh @@ -19,6 +19,7 @@ fi : ${NODE_VERSION="10"} : ${RUBY_VERSION="2.6.2"} : ${YARN_VERSION="1.13.0"} +: ${PNPM_VERSION="7.13.4"} : ${NPM_VERSION=""} : ${HUGO_VERSION="0.54.0"} : ${PHP_VERSION="5.6"} @@ -47,6 +48,7 @@ docker run --rm \ -e NODE_VERSION \ -e RUBY_VERSION \ -e YARN_VERSION \ + -e PNPM_VERSION \ -e NPM_VERSION \ -e HUGO_VERSION \ -e PHP_VERSION \ diff --git a/tests/node/base.bats b/tests/node/base.bats index f76a050a..c62cca82 100644 --- a/tests/node/base.bats +++ b/tests/node/base.bats @@ -10,8 +10,9 @@ setup() { source_nvm } +NODE_VERSION=16 + @test 'node version ${NODE_VERSION} is installed and available at startup' { - NODE_VERSION=16 run node --version assert_success assert_output --partial $NODE_VERSION diff --git a/tests/node/pnpm.bats b/tests/node/pnpm.bats new file mode 100644 index 00000000..38bc7bef --- /dev/null +++ b/tests/node/pnpm.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +load "../helpers.sh" + +load '../../node_modules/bats-support/load' +load '../../node_modules/bats-assert/load' +load '../../node_modules/bats-file/load' + +setup() { + TMP_DIR=$(setup_tmp_dir) + set_fixture_as_repo 'simple-node' "$TMP_DIR" + + # Load functions + load '../../run-build-functions.sh' + + source_nvm +} + +teardown() { + rm -rf "$TMP_DIR" + # Return to original dir + cd - || return +} + +@test 'run_pnpm with a new pnpm version' { + local newPnpmVersion=6.32.20 + # We can't use bats `run` because environmental changes aren't persisted + # We also need to ignore the exit code as the test env is set to return on any non-zero exit code, which we use for + # our workspaces checks + run_pnpm $newPnpmVersion || true > /dev/null 2>&1 + + # New pnpm binary is set in PATH + run pnpm --version + assert_output $newPnpmVersion +} + +@test 'run_pnpm installs a new pnpm version if different from the one installed, installs deps and creates cache dir' { + local newPnpmVersion=6.32.20 + run run_pnpm $newPnpmVersion + assert_success + assert_output --partial "Installing NPM modules using PNPM version $newPnpmVersion" + assert_dir_exist '/opt/buildhome/.pnpm-store' + + # The cache dir is actually being used + assert_dir_exist "/opt/buildhome/.pnpm-store/v3" +} + +@test 'run_pnpm should exit and fail when trying to be used on a to old node version' { + local newPnpmVersion=6.32.20 + + run bash -c ". '/opt/build-bin/run-build-functions.sh' && install_node 12 && run_pnpm $newPnpmVersion" + assert_failure + assert_output --partial "Error while installing PNPM $newPnpmVersion" +} diff --git a/tests/node/yarn.bats b/tests/node/yarn.bats index a424ece5..1b9b581b 100644 --- a/tests/node/yarn.bats +++ b/tests/node/yarn.bats @@ -27,6 +27,17 @@ teardown() { cd - || return } +@test 'run_yarn installs a new yarn version if different from the one installed, installs deps and creates cache dir' { + local newYarnVersion=1.21.0 + run run_yarn $newYarnVersion + assert_success + assert_output --partial "Installing yarn at version $newYarnVersion" + assert_dir_exist $YARN_CACHE_DIR + + # The cache dir is actually being used + assert_dir_exist "$YARN_CACHE_DIR/v6" +} + @test 'run_yarn with a new yarn version correctly sets the new yarn binary in PATH' { local newYarnVersion=1.21.0 # We can't use bats `run` because environmental changes aren't persisted @@ -39,17 +50,6 @@ teardown() { assert_output $newYarnVersion } -@test 'run_yarn installs a new yarn version if different from the one installed, installs deps and creates cache dir' { - local newYarnVersion=1.21.0 - run run_yarn $newYarnVersion - assert_success - assert_output --partial "Installing yarn at version $newYarnVersion" - assert_dir_exist $YARN_CACHE_DIR - - # The cache dir is actually being used - assert_dir_exist "$YARN_CACHE_DIR/v6" -} - @test 'run_yarn allows passing multiple yarn flags via YARN_FLAGS env var to yarn install' { YARN_FLAGS="--no-default-rc --verbose" run run_yarn $YARN_DEFAULT_VERSION @@ -72,3 +72,32 @@ teardown() { assert_dir_exist "$YARN_CACHE_DIR/v6" assert_dir_not_exist "$tmpCacheDir" } + +@test 'run_yarn with a new yarn version correctly on a newer node version' { + local newYarnVersion=1.21.0 + + # We can't use bats `run` because environmental changes aren't persisted + # We also need to ignore the exit code as the test env is set to return on any non-zero exit code, which we use for + # our workspaces checks + install_node "18" || true > /dev/null 2>&1 + run_yarn $newYarnVersion || true > /dev/null 2>&1 + + # New yarn binary is set in PATH + run yarn --version + assert_output $newYarnVersion +} + +# run this test as last one as it changes the node version and would affect the other tests +@test 'run_yarn with a new yarn version correctly on an old node version without corepack' { + local newYarnVersion=1.21.0 + + # We can't use bats `run` because environmental changes aren't persisted + # We also need to ignore the exit code as the test env is set to return on any non-zero exit code, which we use for + # our workspaces checks + install_node "12" || true > /dev/null 2>&1 + run_yarn $newYarnVersion || true > /dev/null 2>&1 + + # New yarn binary is set in PATH + run yarn --version + assert_output $newYarnVersion +}