diff --git a/images/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile index a9f65425..0a57bd23 100644 --- a/images/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=python-3.9.5 +ARG PYTHON_VERSION=python-3.11 FROM jupyter/datascience-notebook:$PYTHON_VERSION # coerce change in all notebook USER root @@ -48,7 +48,7 @@ COPY /test/test_r_func.R /opt/manual_tests USER jovyan -ARG PY_VER_SHORT=3.9 +ARG PY_VER_SHORT=3.11 ARG JUPYTERHUB_VERSION=3.0.0 # mistune added for nbgrader issues diff --git a/images/scipy-ml-notebook/Dockerfile b/images/scipy-ml-notebook/Dockerfile index 82e4debc..8fafee30 100644 --- a/images/scipy-ml-notebook/Dockerfile +++ b/images/scipy-ml-notebook/Dockerfile @@ -81,24 +81,24 @@ RUN pip install datascience \ # Beware of potentially needing to update these if we update the drivers. RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 && \ pip cache purge && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_cnn_infer.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcublasLt.so.11 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_adv_infer.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_adv_train.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_cnn_train.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_ops_infer.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcudnn_ops_train.so.8 && \ - rm /opt/conda/lib/python3.9/site-packages/torch/lib/libcublas.so.11 + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_cnn_infer.so.8 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcublasLt.so.11 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_adv_infer.so.8 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_adv_train.so.8 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_cnn_train.so.8 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_ops_infer.so.8 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcudnn_ops_train.so.8 && \ + rm /opt/conda/lib/python3.11/site-packages/torch/lib/libcublas.so.11 USER $NB_UID:$NB_GID ENV PATH=${PATH}:/usr/local/nvidia/bin:/opt/conda/bin -#ENV CUDNN_PATH=/opt/conda/lib/python3.9/site-packages/nvidia/cudnn +#ENV CUDNN_PATH=/opt/conda/lib/python3.11/site-packages/nvidia/cudnn # starts like this: /opt/conda/pkgs/cudnn-8.6.0.163-pypi_0 8.8.1.3-pypi_0/lib/:/opt/conda/pkgs/cudatoolkit-11.8.0-h37601d7_11/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 -# need to have the end result of running 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/conda/lib/python3.9/site-packages/nvidia/cudnn/lib' +# need to have the end result of running 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/conda/lib/python3.11/site-packages/nvidia/cudnn/lib' # then the gpu can be detected via CLI. -#ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/conda/lib/python3.9/site-packages/nvidia/cudnn/lib +#ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/conda/lib/python3.11/site-packages/nvidia/cudnn/lib # Do some CONDA/CUDA stuff # Copy libdevice file to the required path diff --git a/images/scipy-ml-notebook/cudnn_env_vars.sh b/images/scipy-ml-notebook/cudnn_env_vars.sh index 04663c75..58582083 100644 --- a/images/scipy-ml-notebook/cudnn_env_vars.sh +++ b/images/scipy-ml-notebook/cudnn_env_vars.sh @@ -2,5 +2,5 @@ # set systemwide env variables for Conda package: cudnn CUDNN_PATH=$(dirname $(python -c "import nvidia.cudnn;print(nvidia.cudnn.__file__)")) -#export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/conda/lib/python3.9/site-packages/nvidia/cudnn/lib +#export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/conda/lib/python3.11/site-packages/nvidia/cudnn/lib export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${CUDNN_PATH}/lib diff --git a/images/scipy-ml-notebook/manual_tests/tensorflow_mtest.ipynb b/images/scipy-ml-notebook/manual_tests/tensorflow_mtest.ipynb index cc8dd2fc..e08a5cf2 100644 --- a/images/scipy-ml-notebook/manual_tests/tensorflow_mtest.ipynb +++ b/images/scipy-ml-notebook/manual_tests/tensorflow_mtest.ipynb @@ -130,7 +130,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": ".5" } }, "nbformat": 4, diff --git a/images/scipy-ml-notebook/tensorrt_env_vars.sh b/images/scipy-ml-notebook/tensorrt_env_vars.sh index c1821d14..d255f57b 100644 --- a/images/scipy-ml-notebook/tensorrt_env_vars.sh +++ b/images/scipy-ml-notebook/tensorrt_env_vars.sh @@ -1,4 +1,4 @@ #!/bin/bash # put tensorrt lib into LD_LIBRARY_PATH -export LD_LIBRARY_PATH="/opt/conda/lib/python3.9/site-packages/tensorrt:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="/opt/conda/lib/python3.11/site-packages/tensorrt:$LD_LIBRARY_PATH" diff --git a/images/spec.yml b/images/spec.yml index 0c84a55a..193ad7dc 100644 --- a/images/spec.yml +++ b/images/spec.yml @@ -4,8 +4,8 @@ images: datascience-notebook: image_name: ghcr.io/ucsd-ets/datascience-notebook build_args: - PYTHON_VERSION: python-3.9.5 - PY_VER_SHORT: "3.9" + PYTHON_VERSION: python-3.11 + PY_VER_SHORT: "3.11" JUPYTERHUB_VERSION: 1.4.1 info_cmds: [PY_VER, PIP_LIST, CONDA_INFO, CONDA_LIST, APT_PKG_LIST] diff --git a/scripts/docker_adapter.py b/scripts/docker_adapter.py index e5e8f600..118bf79a 100644 --- a/scripts/docker_adapter.py +++ b/scripts/docker_adapter.py @@ -69,7 +69,7 @@ def build(node: Node) -> Tuple[bool, str]: cache_from=[node.full_image_name, node.stable_image_name] ): # line is of type dict - content_str = line.get('stream', '').strip() # sth like 'Step 1/20 : ARG PYTHON_VERSION=python-3.9.5' + content_str = line.get('stream', '').strip() # sth like 'Step 1/20 : ARG PYTHON_VERSION=python-3.11' if content_str: # if not empty string # time each major step (Step 1/23 : xxx) if content_str[:4] == "Step": diff --git a/scripts/runner.py b/scripts/runner.py index 3de26a78..77f02d04 100644 --- a/scripts/runner.py +++ b/scripts/runner.py @@ -347,7 +347,7 @@ def build_and_test_containers( 'images': { 'datascience-notebook': { 'build_args': { - 'PYTHON_VERSION': 'python-3.9.5' + 'PYTHON_VERSION': 'python-3.11' }, 'info_cmds': [ 'PY_VER',