update docker image build to use micromamba and garbage collection to…

… reduce image size

resources/containers/base-image/Dockerfile

@@ -1,5 +1,5 @@
 # ************************* Base Dockerfile *************************
-# Installs the conda/spack environments and system dependencies
+# Installs the micromamba/spack environments and system dependencies
 # Note: This will take a long time to build due to the spack 
 # installations
 FROM amazonlinux:2
@@ -8,9 +8,9 @@ RUN mkdir /home/al2/
 RUN mkdir /home/al2/install-scripts 
 WORKDIR /home/al2/
 
-# install conda
-ADD install-scripts/install-conda.sh /home/al2/install-scripts/install-conda.sh
-RUN source install-scripts/install-conda.sh
+# install micromamba
+ADD install-scripts/install-micromamba.sh /home/al2/install-scripts/install-micromamba.sh
+RUN source install-scripts/install-micromamba.sh
 
 # install spack
 ADD install-scripts/install-spack.sh /home/al2/install-scripts/install-spack.sh

resources/containers/base-image/README.md

@@ -2,7 +2,7 @@
 This directory contains the files necessary to build a docker image with all the necessary dependencies for running the IMI. The build time for this image takes a long time to build the spack dependencies, so it is meant to contain the software dependencies that take significant time to build and are not regularly updated (gcc, esmf, conda, etc). This is then used as a base-image for the imi Dockerfile, which adds the relevant imi source code to the image.
 
 # Dockerfile
-Dockerfiles contain a list of instructions to build an image with the necessary dependencies preinstalled. The Dockerfile starts with the amazonlinux:2 image available on dockerhub -- this image already contains the aws-cli. The Dockerfile then runs several installation scripts (located in the install-scripts/ directory) to install conda, spack, slurm, and any other necessary dependencies
+Dockerfiles contain a list of instructions to build an image with the necessary dependencies preinstalled. The Dockerfile starts with the amazonlinux:2 image available on dockerhub -- this image already contains the aws-cli. The Dockerfile then runs several installation scripts (located in the install-scripts/ directory) to install micromamba (conda), spack, slurm, and any other necessary dependencies
 
 ## Prerequisites
 [Docker](https://www.docker.com/) must be installed to run or build IMI docker containers.

resources/containers/base-image/install-scripts/install-imi-env.sh

@@ -2,13 +2,15 @@
 
 set -e
 # install python dependencies for imi into conda environment
-# Activate conda env
+# Activate micromamba and spack
 source /etc/bashrc
 spack env activate compute_env
 
-# install mamba for faster installs
-conda install -c conda-forge mamba
-
 # install the environment
-mamba env create -f install-scripts/imi_env.yml
-conda activate imi_env
+micromamba env create -f install-scripts/imi_env.yml
+micromamba clean -a
+micromamba activate imi_env
+
+# Also install yq into this environment
+wget https://github.com/mikefarah/yq/releases/download/v4.32.2/yq_linux_amd64 -O /opt/micromamba/envs/imi_env/bin/yq  \
+    && chmod +x /opt/micromamba/envs/imi_env/bin/yq

resources/containers/base-image/install-scripts/install-micromamba.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# This component installs micromamba
+# which is smaller and faster than conda
+MicromambaInstallURL="https://micro.mamba.pm/api/micromamba/linux-64/latest"
+yum install -y bzip2 tar
+mkdir -p /opt/micromamba/bin/
+curl -Ls $MicromambaInstallURL | tar -xvj bin/micromamba
+mv bin/micromamba /opt/micromamba/bin/micromamba
+rm -rf bin
+
+# Configuration in bashrc file
+echo "export MAMBA_ROOT_PREFIX=/opt/micromamba" >> /etc/bashrc
+echo 'eval "$(/opt/micromamba/bin/micromamba shell hook -s posix)"' >> /etc/bashrc
+
+# set alias to use conda and micromamba interchangeably
+echo "alias conda='micromamba'" >> /etc/bashrc

resources/containers/base-image/install-scripts/install-spack-env.sh

@@ -24,6 +24,9 @@ spack load $SpackCompiler
 spack env activate $SpackEnvironmentName
 spack install --fail-fast --show-log-on-error
 spack clean --all
+
+# garbage collector to reduce the size of spack install
+spack gc -Ey
 
 # AutomaticallyLoadEnvironmentOnLogin
 echo spack env activate $SpackEnvironmentName >> /etc/bashrc

resources/containers/base-image/install-scripts/install-spack.sh

@@ -5,7 +5,8 @@ set -x
 set -e # exit 1 if error
 
 # Spack version to install
-SpackVersion="v0.21.0"
+# TODO: replace with SpackVersion="v0.22.0" when released
+SpackVersion="561da58cea1475927b22b95aa7a6b567ef1105f3"
 # Spack install spec for desired compiler
 SpackCompiler="[email protected]"
 # Name of spack env
@@ -15,6 +16,10 @@ SpackEnvironmentFile="geoschem_deps-gnu-openmpi-102.yaml"
 # InstallSpack
 source /etc/bashrc
 
+# Spack needs python 3.6 or higher
+micromamba create -n py39 python=3.9.1 -c anaconda
+micromamba activate py39
+
 yum install -y git curl curl-devel texinfo
 yum groupinstall -y "Development tools"
 umask 022
@@ -30,6 +35,7 @@ spack install --fail-fast $SpackCompiler
 spack load $SpackCompiler
 spack compiler find --scope system
 
-# Add spack setup to bashrc
+# Add spack setup to bashrc ensuring we activate py39 first
+echo "micromamba activate py39" >> /etc/bashrc
 echo "export SPACK_ROOT=/opt/spack" >> /etc/bashrc
 echo . /opt/spack/share/spack/setup-env.sh >> /etc/bashrc

resources/containers/base-image/install-scripts/install-system-deps.sh

@@ -2,14 +2,12 @@
 
 set -e # exit 1 if error
 
-yum update
-yum install -y emacs wget time jq less glibc which sudo curl
+yum -y update
+yum install -y vim wget time jq less glibc which sudo curl
 
 # install aws-cli
 curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
 unzip awscliv2.zip
-wget https://github.com/mikefarah/yq/releases/download/v4.32.2/yq_linux_amd64 -O /opt/conda/envs/py39/bin/yq  \
-    && chmod +x /opt/conda/envs/py39/bin/yq
 ./aws/install
 
 # cleanup

resources/containers/entrypoint.sh

@@ -19,7 +19,7 @@ slurmctld
 slurmd
 
 spack env activate compute_env
-conda activate imi_env
+micromamba activate imi_env
 
 # set up environment variables for GEOS-Chem
 export NETCDF_HOME=$(spack location -i netcdf-c)