diff --git a/images/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile index 97358f1e..0e6f4c52 100644 --- a/images/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -1,5 +1,5 @@ -ARG PYTHON_VERSION=python-3.9.5 -FROM jupyter/datascience-notebook:$PYTHON_VERSION +ARG PYTHON_VERSION=python-3.11.8 +FROM quay.io/jupyter/datascience-notebook:$PYTHON_VERSION # coerce change in all notebooks USER root @@ -48,19 +48,19 @@ COPY /test/test_r_func.R /opt/manual_tests USER jovyan -ARG PY_VER_SHORT=3.9 +ARG PY_VER_SHORT=3.11 ARG JUPYTERHUB_VERSION=3.0.0 # mistune added for nbgrader issues -# coerce pandas to 1.5.3. 2.0 released april 3rd and currently breaks tests. +# jupyterlab_rise must be below 0.40.0 while jupyterlab is not > v4.0.0 RUN /usr/share/datahub/scripts/install-python-all.sh && \ pip install pandas==2.0.3 'mistune>=2' --upgrade && \ pip install nltk \ pip install statsmodels==0.14.0 \ nbconvert==7.2.1 \ jupyterhub==$JUPYTERHUB_VERSION && \ - mamba install -c conda-forge rise -y && \ + mamba install -c conda-forge "jupyterlab_rise<0.40.0" -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ conda clean --all && \ @@ -87,7 +87,7 @@ RUN pip install dpkt \ fix-permissions /home/$NB_USER && \ pip cache purge -RUN conda clean -tipsy +RUN conda clean -t -p -i -y # ENV SHELL=/bin/bash Weird, this line is in the datascience and rstudio notebooks but not scipyml diff --git a/images/scipy-ml-notebook/Dockerfile b/images/scipy-ml-notebook/Dockerfile index 91707b89..ce530229 100644 --- a/images/scipy-ml-notebook/Dockerfile +++ b/images/scipy-ml-notebook/Dockerfile @@ -10,9 +10,9 @@ USER root ARG LIBNVINFER=7.2.2 LIBNVINFER_MAJOR_VERSION=7 # Python/Mamba deps -ARG CUDA_VERSION=11.8 CUDNN_VERSION=8.6.0.163 \ - PROTOBUF_VERSION=3.20.3 TENSORFLOW_VERSION=2.13.1 KERAS_VERSION=2.13.1 TENSORRT_VERSION=8.5.3.1 \ - TORCH_VERSION=2.1.2 +ARG CUDA_VERSION=11.8 CUDNN_VERSION=8.7.0.84 \ + PROTOBUF_VERSION=3.20.3 TENSORFLOW_VERSION=2.15.0 KERAS_VERSION=2.15.0 TENSORRT_VERSION=8.6.1 \ + TORCH_VERSION=2.2.1 # apt deps RUN apt-get update && \ @@ -65,7 +65,6 @@ RUN pip install --no-cache-dir protobuf==$PROTOBUF_VERSION # tensorrt installed to fix not having libnvinfer that has caused tensorflow issues. RUN pip install opencv-contrib-python-headless \ opencv-python \ - datascience \ nvidia-cudnn-cu11==$CUDNN_VERSION \ tensorflow==$TENSORFLOW_VERSION \ tensorflow-datasets \ @@ -74,20 +73,13 @@ RUN pip install opencv-contrib-python-headless \ fix-permissions /home/$NB_USER && \ pip cache purge -# Some notes as to why we are using pip AND mamba: -# - ntlk_data cannot currently be installed with mamba (while we have python 3.8). -# - datascience cannot be installed with mamba -# - The latest version of pytables, a dependency, only supports python 3.9 and up. -# - The latest compatible version (3.6.1) seems to be broken. -# - pytables is necessary, otherwise nltk will install out-of-date package -# - pytables on conda == tables on pip (???) -# - without pytables explicitly defined, version 3.6 will be installed (which seems to be broken when testing the import) RUN mamba install pyqt \ scapy \ + datascience \ nltk_data \ pycocotools \ pillow \ - keras=$KERAS_VERSION \ + keras=$KERAS_VERSION -y \ -c conda-forge && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -99,15 +91,16 @@ RUN mamba install pyqt \ RUN pip install torch==$TORCH_VERSION torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ - mamba clean -a -y && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_cnn_infer.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcublasLt.so.11 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_adv_infer.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_adv_train.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_cnn_train.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_ops_infer.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_ops_train.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcublas.so.11 + mamba clean -a -y + # && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_cnn_infer.so.8 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcublasLt.so.11 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_adv_infer.so.8 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_adv_train.so.8 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_cnn_train.so.8 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_ops_infer.so.8 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_ops_train.so.8 && \ + #rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcublas.so.11 USER $NB_UID:$NB_GID ENV PATH=${PATH}:/usr/local/nvidia/bin:/opt/conda/bin diff --git a/images/scipy-ml-notebook/tensorrt_env_vars.sh b/images/scipy-ml-notebook/tensorrt_env_vars.sh index c1821d14..d255f57b 100644 --- a/images/scipy-ml-notebook/tensorrt_env_vars.sh +++ b/images/scipy-ml-notebook/tensorrt_env_vars.sh @@ -1,4 +1,4 @@ #!/bin/bash # put tensorrt lib into LD_LIBRARY_PATH -export LD_LIBRARY_PATH="/opt/conda/lib/python3.9/site-packages/tensorrt:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="/opt/conda/lib/python3.11/site-packages/tensorrt:$LD_LIBRARY_PATH" diff --git a/images/spec.yml b/images/spec.yml index 0c84a55a..8c76ecd0 100644 --- a/images/spec.yml +++ b/images/spec.yml @@ -4,8 +4,8 @@ images: datascience-notebook: image_name: ghcr.io/ucsd-ets/datascience-notebook build_args: - PYTHON_VERSION: python-3.9.5 - PY_VER_SHORT: "3.9" + PYTHON_VERSION: python-3.11.8 + PY_VER_SHORT: "3.11" JUPYTERHUB_VERSION: 1.4.1 info_cmds: [PY_VER, PIP_LIST, CONDA_INFO, CONDA_LIST, APT_PKG_LIST] diff --git a/scripts/runner.py b/scripts/runner.py index 3de26a78..d578bb01 100644 --- a/scripts/runner.py +++ b/scripts/runner.py @@ -347,7 +347,7 @@ def build_and_test_containers( 'images': { 'datascience-notebook': { 'build_args': { - 'PYTHON_VERSION': 'python-3.9.5' + 'PYTHON_VERSION': 'python-3.11.8' }, 'info_cmds': [ 'PY_VER',