diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index b84b76dc..57c30ef2 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -11,6 +11,10 @@ on: type: boolean description: 'dry-run: retrieve original image names ONLY' required: true + skipGPUTest: + type: boolean + description: 'skip-gpu-test: Skips testing via the GPU. Never use this unless dsmlp-login is encountering issues!' + required: true jobs: @@ -40,6 +44,7 @@ jobs: gpu_test: uses: ./.github/workflows/test_gpu.yml + if: ${{ github.event.inputs.skipGPUTest == 'false' }} with: originalTag: ${{ github.event.inputs.originalTag }} dryRun: ${{ github.event.inputs.dryRun == 'true' }} @@ -47,9 +52,8 @@ jobs: inherit tagging: - needs: - - gpu_test - - start-runner + needs: start-runner + if: ${{ github.event.inputs.skipGPUTest == 'true' || needs.gpu_test.result == 'success' }} runs-on: ${{ needs.start-runner.outputs.label }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tag_global_stable.yml b/.github/workflows/tag_global_stable.yml index 394ae3e9..23f1ae2c 100644 --- a/.github/workflows/tag_global_stable.yml +++ b/.github/workflows/tag_global_stable.yml @@ -24,6 +24,35 @@ jobs: docker image prune -a -f docker container prune -f + - name: Free disk space + run: | + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + sudo rm -rf \ + /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /usr/lib/jvm || true + echo "some directories deleted" + sudo apt install aptitude -y >/dev/null 2>&1 + sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ + esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ + google-cloud-sdk imagemagick \ + libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ + mercurial apt-transport-https mono-complete libmysqlclient \ + unixodbc-dev yarn chrpath libssl-dev libxft-dev \ + libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ + snmp pollinate libpq-dev postgresql-client powershell ruby-full \ + sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ + -y -f >/dev/null 2>&1 + sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 + sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 + sudo apt-get autoremove -y >/dev/null 2>&1 + sudo apt-get autoclean -y >/dev/null 2>&1 + echo "some packages purged" + - name: Set up Python 3.8 uses: actions/setup-python@v3 with: diff --git a/.github/workflows/test_gpu.yml b/.github/workflows/test_gpu.yml index b3aa8c70..f75ca337 100644 --- a/.github/workflows/test_gpu.yml +++ b/.github/workflows/test_gpu.yml @@ -59,7 +59,7 @@ jobs: echo "About to SSH." - output=$(sshpass -p ${{ secrets.DSMLP_LOGIN_PASSWORD }} ssh grader-test-01@dsmlp-login.ucsd.edu '/opt/launch-sh/bin/launch.sh -n 22 -i ghcr.io/ucsd-ets/scipy-ml-notebook:${{ github.event.inputs.originalTag }} -P Always -g 1 -c 4 -m 32 -f pytest /opt/workflow_tests') + output=$(sshpass -p ${{ secrets.DSMLP_LOGIN_PASSWORD }} ssh grader-test-01@dsmlp-login.ucsd.edu '/opt/launch-sh/bin/launch.sh -i ghcr.io/ucsd-ets/scipy-ml-notebook:${{ github.event.inputs.originalTag }} -P Always -g 1 -c 4 -m 32 -f pytest /opt/workflow_tests') echo "$output" if [[ "$output" == *"FAILURES"* ]]; then diff --git a/images/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile index daf00ce3..69c3c5b4 100644 --- a/images/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -1,7 +1,7 @@ # If you are building manually, pass PYTHON_VERSION/PY_VER_SHORT/JUPYTERHUB_VERSION with --build-arg -ARG PYTHON_VERSION=python-3.11.8 +ARG PYTHON_VERSION=python-3.11.10 ARG PY_VER_SHORT=3.11 -ARG JUPYTERHUB_VERSION=4.1.5 +ARG JUPYTERHUB_VERSION=4.1.6 # Jupyter has changed where it stores its images FROM quay.io/jupyter/datascience-notebook:$PYTHON_VERSION @@ -34,6 +34,9 @@ RUN apt-get update -y && \ apt-utils \ jq \ build-essential \ + iputils-ping \ + emacs \ + zip \ p7zip-full && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ chmod g-s /usr/bin/screen && \ @@ -69,8 +72,8 @@ USER jovyan # Python/Mamba Deps ## Package versions -ARG JUPYTERSERVER_VERSION=2.14.2 NBGRADER_VERSION=0.9.3 JUPYTERLAB_VERSION=4.2.5 NBCONVERT_VERSION=7.16.4 NOTEBOOK_VERSION=7.2.1 NBCLASSIC_VERSION=1.1.0 -ARG PANDAS_VERSION=2.2.2 STATSMODELS_VERSION=0.14.2 BOTTLENECK_VERSION=1.3.6 NUMEXPR_VERSION=2.8.4 +ARG JUPYTERSERVER_VERSION=2.14.2 NBGRADER_VERSION=0.9.3 JUPYTERLAB_VERSION=4.3.3 NBCONVERT_VERSION=7.16.4 NOTEBOOK_VERSION=7.2.1 NBCLASSIC_VERSION=1.1.0 +ARG PANDAS_VERSION=2.2.3 STATSMODELS_VERSION=0.14.4 BOTTLENECK_VERSION=1.4.2 NUMEXPR_VERSION=2.10.2 # Install essential+datascience pip packages ## mistune added for nbgrader issues @@ -88,7 +91,7 @@ RUN mamba install -c conda-forge jupyterhub=$JUPYTERHUB_VERSION jupyter_server=$ # (TODO: Re-enable collab once RTC is fixed) mamba install -c conda-forge jupyterlab_rise jupyter_server_terminals jupyter-collaboration && \ mamba install -c conda-forge jupyterlab_rise jupyter_server_terminals && \ mamba install -c conda-forge jupyterlab-latex jupyterlab-git jupyterlab-fasta jupyterlab-geojson && \ - mamba install -c conda-forge jupyter_scheduler jupyterlab_code_formatter isort black jupyter-archive jupyterlab-lsp python-lsp-server r-languageserver && \ + # mamba install -c conda-forge jupyter_scheduler jupyterlab_code_formatter isort black jupyter-archive jupyterlab-lsp python-lsp-server r-languageserver && \ mamba install -c conda-forge nbconvert=$NBCONVERT_VERSION nbgrader=$NBGRADER_VERSION && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index db504d2f..ff32beed 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -7,7 +7,7 @@ USER root # Ubuntu 22 setup with v 2024.04.2-764 ## Follow instructions at https://www.rstudio.com/products/rstudio/download-server/ -ENV RSTUDIO_PKG=rstudio-server-2024.04.2-764-amd64.deb +ENV RSTUDIO_PKG=rstudio-server-2024.09.1-394-amd64.deb ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/${RSTUDIO_PKG} ## rstudio installation expects R to live in /usr/bin, /bin/, etc. @@ -25,15 +25,39 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy -RUN mkdir -p /etc/rstudio && echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf +RUN mkdir -p /etc/rstudio && \ + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf -## Shim in our nss-wrapper so that user getpwent() lookups succeed -RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site +## Shim in our LD_LIBRARY_PATH for OpenSSL + nss-wrapper so that user getpwent() lookups succeed +RUN ( echo LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu"; echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash \ - LD_LIBRARY_PATH="/usr/lib/R/lib:/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server:/opt/conda/lib/R/lib" + SHELL=/bin/bash + +### OpenSSL Conda Workarounds +# When we upgraded to Python3.11/JLab 4 upstreamed from Jupyter, our distro was also upgraded from Ubuntu 20 to 22. +# This introduced a new issue where RStudio would prefer /opt/conda/lib's OpenSSL instead of /usr/lib when trying to download packages using internet.so (internal RStudio lib). +# https://github.com/rstudio/rstudio/issues/14060 - the fix described was setting LD_LIBRARY_PATH to /opt/conda/lib:/usr/lib/x86_64-linux-gnu (globally or just for RStudio). +# However, this fix is problematic because many executables such as SSH*/git rely on the OpenSSL version being installed by apt (since they are installed by apt). +# Post-fix, when you tried to launch ssh/git in the RStudio Terminal, you would receieve an "OpenSSL mismatch" error. +# To fix this, we simply added a bash wrapper for all affected binaries that forces them to always use the correct LD_LIBRARY_PATH (apt). +# If another binary encounters a "OpenSSL mismatch error," add it here. +# Hopefully a better fix can be found - the core issue here seems to be conda and its obtrusiveness + +# List of binaries you want to wrap +ENV FORCED_BINARIES="ssh ssh-add ssh-agent ssh-copy-id ssh-keygen ssh-keyscan sshd git nano" + +# Loop over each binary and apply the wrapper +RUN for bin in $FORCED_BINARIES; do \ + if [ -f /usr/bin/$bin ]; then \ + mv /usr/bin/$bin /usr/bin/$bin.orig && \ + echo '#!/bin/bash' > /usr/bin/$bin && \ + echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/$bin && \ + echo 'exec /usr/bin/'$bin'.orig "$@"' >> /usr/bin/$bin && \ + chmod +x /usr/bin/$bin; \ + fi; \ + done COPY ./lib /opt/k8s-support/lib diff --git a/images/spec.yml b/images/spec.yml index a61ff5b2..8a8aab35 100644 --- a/images/spec.yml +++ b/images/spec.yml @@ -4,9 +4,9 @@ images: datascience-notebook: image_name: ghcr.io/ucsd-ets/datascience-notebook build_args: - PYTHON_VERSION: python-3.11.8 + PYTHON_VERSION: python-3.11.10 PY_VER_SHORT: "3.11" - JUPYTERHUB_VERSION: 4.1.5 + JUPYTERHUB_VERSION: 4.1.6 info_cmds: [PY_VER, PIP_LIST, CONDA_INFO, CONDA_LIST, APT_PKG_LIST] rstudio-notebook: @@ -25,7 +25,7 @@ images: #prepull: false #-- uncomment to disable prepulling behavior for scipy-ml. gives you space on machine in exchange for build time. tag: - prefix: "2024.4" + prefix: "2025.1" all_info_cmds: PY_VER: