feat: add pnpm to build image (#845)

* feat: add pnpm to build image * chore: add PR review * chore: always enable node corepack * chore: restore how we install yarn and check npm version for corepack * chore: cleanup * chore: revert yarn tests * feat: make yarn + corepack working together * Update run-build-functions.sh Co-authored-by: Nicola Aitken <[email protected]> * chore: fix typo Co-authored-by: Nicola Aitken <[email protected]> * chore: fix typo Co-authored-by: Nicola Aitken <[email protected]> * chore: add PR feedback * chore: remove unused function Co-authored-by: Nicola Aitken <[email protected]>
netlify · Oct 17, 2022 · 788d280 · 788d280
1 parent 9289b69
commit 788d280
Show file tree

Hide file tree

Showing 11 changed files with 229 additions and 89 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,2 @@
 tmp/
 node_modules
-package-lock.json
-yarn.lock
diff --git a/Dockerfile b/Dockerfile
@@ -274,29 +274,31 @@ USER root
 #
 ################################################################################
 
-
+# this installer is needed for older node versions where no corepack is available
 RUN curl -o- -L https://yarnpkg.com/install.sh > /usr/local/bin/yarn-installer.sh
 
 ENV NVM_VERSION=0.39.1
 
-# Install node.js, yarn, grunt, bower and elm
+# Install node.js, yarn, grunt, bower
 USER buildbot
 RUN git clone https://github.com/creationix/nvm.git ~/.nvm && \
     cd ~/.nvm && \
     git checkout v$NVM_VERSION && \
     cd /
 
-ENV ELM_VERSION=0.19.1-5
 ENV YARN_VERSION=1.22.19
+ENV PNPM_VERSION=7.13.4
 
 ENV NETLIFY_NODE_VERSION="16"
 
 RUN /bin/bash -c ". ~/.nvm/nvm.sh && \
          nvm install --no-progress $NETLIFY_NODE_VERSION && \
          npm install -g grunt-cli bower && \
-             bash /usr/local/bin/yarn-installer.sh --version $YARN_VERSION && \
-         nvm alias default node && nvm cache clear"
-ENV PATH "/opt/buildhome/.yarn/bin:$PATH"
+         nvm alias default node && \
+         nvm cache clear && \
+         corepack enable && \
+         corepack prepare yarn@$YARN_VERSION --activate && \
+         corepack prepare pnpm@$PNPM_VERSION --activate"
 
 USER root
 

diff --git a/Makefile b/Makefile
@@ -47,7 +47,6 @@ container-test: build-base ## Run a container structure test
 run-local: build-base ## Volume the build scripts and run a bash shell in the build-image
 	docker run --rm -it \
 		-e NETLIFY_BUILD_BASE="/opt/buildhome" \
-		-u root \
 		-v $(PWD)/run-build.sh:/opt/build-bin/build:ro \
 		-v $(PWD)/run-build-functions.sh:/opt/build-bin/run-build-functions.sh:ro \
 		$(image)
diff --git a/focal.yaml b/focal.yaml
@@ -30,7 +30,6 @@ metadataTest:
         /opt/buildhome/.php:\
         /opt/buildhome/.binrc/bin:\
         /opt/buildhome/.deno/bin:\
-        /opt/buildhome/.yarn/bin:\
         /usr/local/rvm/bin:\
         /usr/local/sbin:\
         /usr/local/bin:\
@@ -91,11 +90,13 @@ commandTests:
         echo "Node: $(node --version)"
         echo "Yarn: $(yarn --version)"
         echo "NPM: $(npm --version)"
+        echo "PNPM: $(pnpm --version)"
     expectedOutput:
       - "Node: v16.*.*"
       - "NVM: 0.39.1"
       - "Yarn: 1.22.*"
       - "NPM: 8.*.*"
+      - "PNPM: 7.*.*"
 
   - name: hugo
     command: "hugo"

diff --git a/run-build-functions.sh b/run-build-functions.sh
@@ -132,32 +132,38 @@ restore_node_modules() {
 
 run_yarn() {
   yarn_version=$1
-  if [ -d $NETLIFY_CACHE_DIR/yarn ]
-  then
-    export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH
-  fi
   restore_home_cache ".yarn_cache" "yarn cache"
 
-  if [ $(which yarn) ] && [ "$(yarn --version)" != "$yarn_version" ]
-  then
-    echo "Found yarn version ($(yarn --version)) that doesn't match expected ($yarn_version)"
-    rm -rf $NETLIFY_CACHE_DIR/yarn $HOME/.yarn
-    npm uninstall yarn -g
-  fi
+  if ! [ $(which corepack) ]; then
+    if [ -d $NETLIFY_CACHE_DIR/yarn ]
+    then
+      export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH
+    fi
 
-  if ! [ $(which yarn) ]
-  then
-    echo "Installing yarn at version $yarn_version"
-    rm -rf $HOME/.yarn
-    bash /usr/local/bin/yarn-installer.sh --version $yarn_version
-    mv $HOME/.yarn $NETLIFY_CACHE_DIR/yarn
-    export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH
+    if [ $(which yarn) ] && [ "$(yarn --version)" != "$yarn_version" ]; then
+        echo "Found yarn version ($(yarn --version)) that doesn't match expected ($yarn_version)"
+        rm -rf $NETLIFY_CACHE_DIR/yarn $HOME/.yarn
+        npm uninstall yarn -g
+    fi
+
+    if ! [ $(which yarn) ]; then
+      echo "Installing yarn at version $yarn_version"
+      rm -rf $HOME/.yarn
+      bash /usr/local/bin/yarn-installer.sh --version $yarn_version
+      mv $HOME/.yarn $NETLIFY_CACHE_DIR/yarn
+      export PATH=$NETLIFY_CACHE_DIR/yarn/bin:$PATH
+    fi
+  else
+    # if corepack is installed use it for changing the yarn version
+    if [ "$(yarn --version)" != "$yarn_version" ]; then
+      echo "Installing yarn at version $yarn_version"
+      corepack prepare yarn@$yarn_version --activate
+    fi
   fi
 
   restore_node_modules "yarn"
 
   echo "Installing NPM modules using Yarn version $(yarn --version)"
-  run_npm_set_temp
 
   # Remove the cache-folder flag if the user set any.
   # We want to control where to put the cache
@@ -177,10 +183,38 @@ run_yarn() {
   export PATH=$(yarn bin):$PATH
 }
 
-run_npm_set_temp() {
-  # Make sure we're not limited by space in the /tmp mount
-  mkdir $HOME/tmp
-  npm set tmp $HOME/tmp
+
+run_pnpm() {
+  pnpm_version=$1
+  restore_home_cache ".pnpm-store" "pnpm cache"
+
+  if ! [ $(which corepack) ]; then
+    echo "Error while installing PNPM $pnpm_version"
+    echo "We cannot install the expected version of PNPM ($pnpm_version) as your required Node.js version $NODE_VERSION does not allow that"
+    echo "Please ensure that you use at least Node Version 14.19.0 or greater than 16.9.0"
+
+    exit 1
+  fi
+
+  if [ "$(pnpm --version)" != "$pnpm_version" ]
+  then
+    echo "Found pnpm version ($(pnpm --version)) that doesn't match expected ($pnpm_version)"
+
+    corepack prepare pnpm@$pnpm_version --activate
+  fi
+
+  restore_node_modules "pnpm"
+
+  echo "Installing NPM modules using PNPM version $(pnpm --version)"
+  if pnpm install
+  then
+    echo "NPM modules installed using PNPM"
+  else
+    echo "Error during PNPM install"
+    exit 1
+  fi
+
+  export PATH=$(pnpm bin):$PATH
 }
 
 run_npm() {
@@ -205,7 +239,7 @@ run_npm() {
   if install_deps package.json $NODE_VERSION $NETLIFY_CACHE_DIR/package-sha
   then
     echo "Installing NPM modules using NPM version $(npm --version)"
-    run_npm_set_temp
+
     if npm install ${NPM_FLAGS:+$NPM_FLAGS}
     then
       echo "NPM modules installed"
@@ -219,46 +253,9 @@ run_npm() {
   export PATH=$(npm bin):$PATH
 }
 
-check_python_version() {
-  if source $HOME/python${PYTHON_VERSION}/bin/activate
-  then
-    echo "Python version set to ${PYTHON_VERSION}"
-  else
-    echo "Error setting python version from $1"
-    echo "Please see https://github.com/netlify/build-image/blob/focal/included_software.md for current versions"
-    exit 1
-  fi
-}
+install_node() {
+  local defaultNodeVersion=$1
 
-read_node_version_file() {
-  local nodeVersionFile="$1"
-  NODE_VERSION="$(cat "$nodeVersionFile")"
-  echo "Attempting node version '$NODE_VERSION' from $nodeVersionFile"
-}
-
-install_dependencies() {
-  local defaultNodeVersion=$1 # 16
-  local defaultRubyVersion=$2 # 2.6.2
-  local defaultYarnVersion=$3 # 1.13.0
-  local installGoVersion=$4 # 1.16.4
-  local defaultPythonVersion=$5 # 3.8
-  local featureFlags="$6"
-
-  # Python Version
-  if [ -f runtime.txt ]
-  then
-    PYTHON_VERSION=$(cat runtime.txt)
-    check_python_version "runtime.txt"
-  elif [ -f Pipfile ]
-  then
-    echo "Found Pipfile restoring Pipenv virtualenv"
-    restore_cwd_cache ".venv" "python virtualenv"
-  else
-    PYTHON_VERSION=$defaultPythonVersion
-    check_python_version "the PYTHON_VERSION environment variable"
-  fi
-
-  # Node version
   source $NVM_DIR/nvm.sh
   : ${NODE_VERSION="$defaultNodeVersion"}
 
@@ -296,13 +293,63 @@ install_dependencies() {
     exit 1
   fi
 
+  # if Node.js Corepack is available enable it
+  if [ $(which corepack) ]; then
+    echo "Enabling node corepack"
+    corepack enable
+  fi
+
   if [ -n "$NPM_TOKEN" ]
   then
     if [ ! -f .npmrc ]
     then
       echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
     fi
   fi
+}
+
+check_python_version() {
+  if source $HOME/python${PYTHON_VERSION}/bin/activate
+  then
+    echo "Python version set to ${PYTHON_VERSION}"
+  else
+    echo "Error setting python version from $1"
+    echo "Please see https://github.com/netlify/build-image/blob/focal/included_software.md for current versions"
+    exit 1
+  fi
+}
+
+read_node_version_file() {
+  local nodeVersionFile="$1"
+  NODE_VERSION="$(cat "$nodeVersionFile")"
+  echo "Attempting node version '$NODE_VERSION' from $nodeVersionFile"
+}
+
+install_dependencies() {
+  local defaultNodeVersion=$1 # 16
+  local defaultRubyVersion=$2 # 2.6.2
+  local defaultYarnVersion=$3 # 1.13.0
+  local defaultPnpmVersion=$4 # 7.13.4
+  local installGoVersion=$5 # 1.16.4
+  local defaultPythonVersion=$6 # 3.8
+  local featureFlags="$7"
+
+  # Python Version
+  if [ -f runtime.txt ]
+  then
+    PYTHON_VERSION=$(cat runtime.txt)
+    check_python_version "runtime.txt"
+  elif [ -f Pipfile ]
+  then
+    echo "Found Pipfile restoring Pipenv virtualenv"
+    restore_cwd_cache ".venv" "python virtualenv"
+  else
+    PYTHON_VERSION=$defaultPythonVersion
+    check_python_version "the PYTHON_VERSION environment variable"
+  fi
+
+  # Node version
+  install_node $defaultNodeVersion
 
   # Automatically installed Build plugins
   if [ ! -d "$PWD/.netlify" ]
@@ -527,19 +574,22 @@ install_dependencies() {
 
   # NPM Dependencies
   : ${YARN_VERSION="$defaultYarnVersion"}
+  : ${PNPM_VERSION="$defaultPnpmVersion"}
   : ${CYPRESS_CACHE_FOLDER="./node_modules/.cache/CypressBinary"}
   export CYPRESS_CACHE_FOLDER
 
   if [ -f package.json ]
   then
-    if [ "$NODE_ENV" == "production" ]
-    then
+    if [ "$NODE_ENV" == "production" ];then
       warn "The environment variable 'NODE_ENV' is set to 'production'. Any 'devDependencies' in package.json will not be installed"
     fi
 
-    if [ "$NETLIFY_USE_YARN" = "true" ] || ([ "$NETLIFY_USE_YARN" != "false" ] && [ -f yarn.lock ])
-    then
+    restore_home_cache ".node/corepack" "corepack dependencies"
+
+    if [ "$NETLIFY_USE_YARN" = "true" ] || ([ "$NETLIFY_USE_YARN" != "false" ] && [ -f yarn.lock ]); then
       run_yarn $YARN_VERSION
+    elif [ "$NETLIFY_USE_PNPM" = "true" ] || ([ "$NETLIFY_USE_PNPM" != "false" ] && [ -f pnpm-lock.yaml ]); then
+      run_pnpm $PNPM_VERSION
     else
       run_npm "$featureFlags"
     fi
@@ -707,6 +757,8 @@ cache_artifacts() {
     cache_cwd_directory_fast_copy "target" "rust compile output"
   fi
 
+  cache_home_directory ".node/corepack" "corepack cache"
+  cache_home_directory ".pnpm-store" "pnpm cache"
   cache_home_directory ".yarn_cache" "yarn cache"
   cache_home_directory ".cache/pip" "pip cache"
   cache_home_directory ".cask" "emacs cask dependencies"

diff --git a/run-build.sh b/run-build.sh
@@ -21,11 +21,12 @@ cd $NETLIFY_REPO_DIR
 : ${NODE_VERSION="16"}
 : ${RUBY_VERSION="2.7.2"}
 : ${YARN_VERSION="1.22.19"}
+: ${PNPM_VERSION="7.13.4"}
 : ${GO_VERSION="1.17"}
 : ${PYTHON_VERSION="3.8"}
 
 echo "Installing dependencies"
-install_dependencies $NODE_VERSION $RUBY_VERSION $YARN_VERSION $GO_VERSION $PYTHON_VERSION
+install_dependencies $NODE_VERSION $RUBY_VERSION $YARN_VERSION $PNPM_VERSION $GO_VERSION $PYTHON_VERSION
 
 echo "Installing missing commands"
 install_missing_commands

diff --git a/test-tools/start-image.sh b/test-tools/start-image.sh
@@ -9,6 +9,7 @@ docker run --rm -t -i \
 	-e NPM_VERSION \
 	-e RUBY_VERSION \
 	-e YARN_VERSION \
+  -e PNPM_VERSION \
 	-e HUGO_VERSION \
 	-e PHP_VERSION \
 	-e GO_VERSION \

diff --git a/test-tools/test-build.sh b/test-tools/test-build.sh
@@ -19,6 +19,7 @@ fi
 : ${NODE_VERSION="10"}
 : ${RUBY_VERSION="2.6.2"}
 : ${YARN_VERSION="1.13.0"}
+: ${PNPM_VERSION="7.13.4"}
 : ${NPM_VERSION=""}
 : ${HUGO_VERSION="0.54.0"}
 : ${PHP_VERSION="5.6"}
@@ -47,6 +48,7 @@ docker run --rm \
        -e NODE_VERSION \
        -e RUBY_VERSION \
        -e YARN_VERSION \
+       -e PNPM_VERSION \
        -e NPM_VERSION \
        -e HUGO_VERSION \
        -e PHP_VERSION \

diff --git a/tests/node/base.bats b/tests/node/base.bats
@@ -10,8 +10,9 @@ setup() {
   source_nvm
 }
 
+NODE_VERSION=16
+
 @test 'node version ${NODE_VERSION} is installed and available at startup' {
-  NODE_VERSION=16
   run node --version
   assert_success
   assert_output --partial $NODE_VERSION