From 8d1c15abb21a6bb85fb82a3337664df371bfcf65 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 1 Aug 2023 16:58:31 -0700 Subject: [PATCH 1/6] Fixes exporting correct environment --- dockerfiles/dask-gateway/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/dask-gateway/Makefile b/dockerfiles/dask-gateway/Makefile index e9bf4d8..0a17bc2 100644 --- a/dockerfiles/dask-gateway/Makefile +++ b/dockerfiles/dask-gateway/Makefile @@ -5,7 +5,7 @@ VERSION = 0.1.0 .PHONY: env-export env-export: ARGS = -v $(PWD)/../minimal-notebook/conda-envs:/conda-envs env-export: ENTRYPOINT = /bin/bash -env-export: COMMAND = -c "conda env export -n dask-gateway -f /conda-envs/dask-gateway.yaml" +env-export: COMMAND = -c "conda env export -f /conda-envs/dask-gateway.yaml" env-export: run include ../Makefile From 8ba6b2ddf8491e69b00350190bd8a8139dfe833a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 1 Aug 2023 18:34:30 -0700 Subject: [PATCH 2/6] Adds changelog generation to build --- .gitignore | 1 + dockerfiles/Makefile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1a13c36 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +changelog.md diff --git a/dockerfiles/Makefile b/dockerfiles/Makefile index 774c7c1..bdc80a7 100644 --- a/dockerfiles/Makefile +++ b/dockerfiles/Makefile @@ -4,7 +4,7 @@ NAMESPACE ?= ghcr.io/esgf-nimbus build push run: REPOSITORY = $(lastword $(subst /, ,$(PWD))) -build: +build: changelog docker build $(ARGS) -t $(NAMESPACE)/$(REPOSITORY):$(VERSION) . push: @@ -12,3 +12,6 @@ push: run: docker run -it $(ARGS) $(if $(ENTRYPOINT),--entrypoint $(ENTRYPOINT),) $(NAMESPACE)/$(REPOSITORY):$(VERSION) $(COMMAND) + +changelog: + git log --pretty="%s" . | python -c "import sys;lines=[f'- {x}' if 'Bump' in x else f' - {x}' for x in sys.stdin]; print(''.join(['# Changelog\n'] + lines))" > changelog.md From 6b9fb7cdb0e83479af017b8f6476cec3c1db0f35 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 1 Aug 2023 18:38:12 -0700 Subject: [PATCH 3/6] Adds dask-gateway and changelog --- .gitignore | 1 + dockerfiles/minimal-notebook/Dockerfile | 24 +- .../conda-envs/dask-gateway.yaml | 211 +++++++++++++++++- .../{intro.ipynb => intro.ipynb.tpl} | 21 +- .../minimal-notebook/scripts/install-envs.sh | 12 + .../minimal-notebook/scripts/render-intro.sh | 3 + 6 files changed, 261 insertions(+), 11 deletions(-) rename dockerfiles/minimal-notebook/notebooks/{intro.ipynb => intro.ipynb.tpl} (94%) create mode 100755 dockerfiles/minimal-notebook/scripts/install-envs.sh create mode 100644 dockerfiles/minimal-notebook/scripts/render-intro.sh diff --git a/.gitignore b/.gitignore index 1a13c36..ea8c60f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ changelog.md +**/.ipynb_checkpoints diff --git a/dockerfiles/minimal-notebook/Dockerfile b/dockerfiles/minimal-notebook/Dockerfile index daf1818..fb07708 100644 --- a/dockerfiles/minimal-notebook/Dockerfile +++ b/dockerfiles/minimal-notebook/Dockerfile @@ -28,14 +28,28 @@ RUN groupadd -g 1026 climate && \ pip install nbgitpuller && \ mkdir /notebooks -COPY notebooks /notebooks COPY conda-envs /conda-envs RUN --mount=type=cache,target=/opt/conda/pkgs \ - mamba env create -n dask-gateway -f /conda-envs/dask-gateway.yaml && \ - source /opt/conda/etc/profile.d/conda.sh && \ - conda activate dask-gateway && \ - python -m ipykernel install --name=dask-gateway && \ + mamba install -y \ + xcdat \ + xarray \ + netcdf4 \ + intake \ + intake-esm \ + intake-stac \ + intake-xarray \ + intake-dataframe-catalog + +COPY scripts /scripts + +RUN --mount=type=cache,target=/opt/conda/pkgs \ + source /scripts/install-envs.sh && \ chown -R ${NB_UID}:${NB_GID} /home/jovyan +COPY notebooks /notebooks +COPY changelog.md /changelog.md + +RUN source /scripts/render-intro.sh + USER $NB_USER diff --git a/dockerfiles/minimal-notebook/conda-envs/dask-gateway.yaml b/dockerfiles/minimal-notebook/conda-envs/dask-gateway.yaml index 847b017..16caa4a 100644 --- a/dockerfiles/minimal-notebook/conda-envs/dask-gateway.yaml +++ b/dockerfiles/minimal-notebook/conda-envs/dask-gateway.yaml @@ -1,4 +1,211 @@ -name: dask-gateway +name: base channels: - conda-forge -prefix: /opt/conda/envs/dask-gateway +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - aiohttp=3.8.5=py310h2372a71_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - asttokens=2.2.1=pyhd8ed1ab_0 + - async-timeout=4.0.2=pyhd8ed1ab_0 + - attrs=23.1.0=pyh71513ae_1 + - aws-c-auth=0.7.0=hf8751d9_2 + - aws-c-cal=0.6.0=h93469e0_0 + - aws-c-common=0.8.23=hd590300_0 + - aws-c-compression=0.2.17=h862ab75_1 + - aws-c-event-stream=0.3.1=h9599702_1 + - aws-c-http=0.7.11=hbe98c3e_0 + - aws-c-io=0.13.28=h3870b5a_0 + - aws-c-mqtt=0.8.14=h2e270ba_2 + - aws-c-s3=0.3.13=heb0bb06_2 + - aws-c-sdkutils=0.1.11=h862ab75_1 + - aws-checksums=0.1.16=h862ab75_1 + - aws-crt-cpp=0.20.3=he9c0e7f_4 + - aws-sdk-cpp=1.10.57=hbc2ea52_17 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - blosc=1.21.4=h0f2a231_0 + - bokeh=2.4.3=pyhd8ed1ab_3 + - brotli-python=1.0.9=py310hd8f1fbe_9 + - bzip2=1.0.8=h7f98852_4 + - c-ares=1.19.1=hd590300_0 + - ca-certificates=2023.5.7=hbcca054_0 + - certifi=2023.5.7=pyhd8ed1ab_0 + - cffi=1.15.1=py310h255011f_3 + - cftime=1.6.2=py310hde88566_1 + - charset-normalizer=3.2.0=pyhd8ed1ab_0 + - click=8.1.6=unix_pyh707e725_0 + - cloudpickle=2.2.1=pyhd8ed1ab_0 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.1.3=pyhd8ed1ab_0 + - conda=23.1.0=py310hff52083_0 + - conda-package-handling=2.0.2=pyh38be061_0 + - conda-package-streaming=0.8.0=pyhd8ed1ab_0 + - cryptography=41.0.2=py310h75e40e8_0 + - cytoolz=0.12.2=py310h2372a71_0 + - dask=2023.7.1=pyhd8ed1ab_0 + - dask-core=2023.7.1=pyhd8ed1ab_0 + - dask-gateway=2023.1.1=pyh8af1aa0_0 + - debugpy=1.6.7=py310heca2aa9_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - distributed=2023.7.1=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - fmt=9.1.0=h924138e_0 + - freetype=2.12.1=hca18f0e_1 + - frozenlist=1.4.0=py310h2372a71_0 + - fsspec=2023.6.0=pyh1a96a4e_0 + - gflags=2.2.2=he1b5a44_1004 + - glog=0.6.0=h6f12383_0 + - hdf4=4.2.15=h501b40f_6 + - hdf5=1.14.1=nompi_h4f84152_100 + - icu=72.1=hcb278e6_0 + - idna=3.4=pyhd8ed1ab_0 + - importlib-metadata=6.8.0=pyha770c72_0 + - importlib_metadata=6.8.0=hd8ed1ab_0 + - ipykernel=6.24.0=pyh71e2992_0 + - ipython=8.14.0=pyh41d4057_0 + - jedi=0.18.2=pyhd8ed1ab_0 + - jinja2=3.1.2=pyhd8ed1ab_1 + - jupyter_client=8.3.0=pyhd8ed1ab_0 + - jupyter_core=5.3.1=py310hff52083_0 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.21.1=h659d440_0 + - lcms2=2.15=haa2dc70_1 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libabseil=20230125.3=cxx17_h59595ed_0 + - libaec=1.0.6=hcb278e6_1 + - libarchive=3.6.2=h039dbb9_1 + - libarrow=12.0.1=h657c46f_5_cpu + - libblas=3.9.0=17_linux64_openblas + - libbrotlicommon=1.0.9=h166bdaf_9 + - libbrotlidec=1.0.9=h166bdaf_9 + - libbrotlienc=1.0.9=h166bdaf_9 + - libcblas=3.9.0=17_linux64_openblas + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcurl=8.1.2=hca28451_1 + - libdeflate=1.18=h0b41bf4_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libevent=2.1.12=hf998b51_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.1.0=he5830b7_0 + - libgfortran-ng=13.1.0=h69a702a_0 + - libgfortran5=13.1.0=h15d22d2_0 + - libgomp=13.1.0=he5830b7_0 + - libgoogle-cloud=2.12.0=h840a212_1 + - libgrpc=1.56.2=h3905398_0 + - libiconv=1.17=h166bdaf_0 + - libjpeg-turbo=2.1.5.1=h0b41bf4_0 + - liblapack=3.9.0=17_linux64_openblas + - libmamba=1.4.2=hcea66bb_0 + - libmambapy=1.4.2=py310h1428755_0 + - libnetcdf=4.9.2=nompi_h7e745eb_109 + - libnghttp2=1.52.0=h61bc06f_0 + - libnsl=2.0.0=h7f98852_0 + - libnuma=2.0.16=h0b41bf4_1 + - libopenblas=0.3.23=pthreads_h80387f5_0 + - libpng=1.6.39=h753d276_0 + - libprotobuf=4.23.3=hd1fb520_0 + - libsodium=1.0.18=h36c2ea0_1 + - libsolv=0.7.24=hfc55251_1 + - libsqlite=3.42.0=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-ng=13.1.0=hfd8a6a1_0 + - libthrift=0.18.1=h8fd135c_2 + - libtiff=4.5.1=h8b53f26_0 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp-base=1.3.1=hd590300_0 + - libxcb=1.15=h0b41bf4_0 + - libxml2=2.11.4=h0d562d8_0 + - libzip=1.9.2=hc929e4a_1 + - libzlib=1.2.13=hd590300_5 + - locket=1.0.0=pyhd8ed1ab_0 + - lz4=4.3.2=py310h0cfdcf0_0 + - lz4-c=1.9.4=hcb278e6_0 + - lzo=2.10=h516909a_1000 + - mamba=1.4.2=py310h51d5547_0 + - markupsafe=2.1.3=py310h2372a71_0 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - msgpack-python=1.0.5=py310hdf3cbec_0 + - multidict=6.0.4=py310h1fa729e_0 + - ncurses=6.4=hcb278e6_0 + - nest-asyncio=1.5.6=pyhd8ed1ab_0 + - netcdf4=1.6.4=nompi_py310h6f5dce6_101 + - numpy=1.25.1=py310ha4c1d20_0 + - openjpeg=2.5.0=hfec8fc6_2 + - openssl=3.1.1=hd590300_1 + - orc=1.9.0=h385abfd_1 + - packaging=23.1=pyhd8ed1ab_0 + - pandas=2.0.3=py310h7cbd5c2_1 + - parso=0.8.3=pyhd8ed1ab_0 + - partd=1.4.0=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pillow=10.0.0=py310h582fbeb_0 + - pip=23.1.2=pyhd8ed1ab_0 + - platformdirs=3.9.1=pyhd8ed1ab_0 + - pluggy=1.2.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - psutil=5.9.5=py310h1fa729e_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pyarrow=12.0.1=py310h0576679_5_cpu + - pybind11-abi=4=hd8ed1ab_3 + - pycosat=0.6.4=py310h5764c6d_1 + - pycparser=2.21=pyhd8ed1ab_0 + - pygments=2.15.1=pyhd8ed1ab_0 + - pyopenssl=23.2.0=pyhd8ed1ab_1 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.10.12=hd12c33a_0_cpython + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python-tzdata=2023.3=pyhd8ed1ab_0 + - python_abi=3.10=3_cp310 + - pytz=2023.3=pyhd8ed1ab_0 + - pyyaml=6.0=py310h5764c6d_5 + - pyzmq=25.1.0=py310h5bbb5d0_0 + - rdma-core=28.9=h59595ed_1 + - re2=2023.03.02=h8c504da_0 + - readline=8.2=h8228510_1 + - reproc=14.2.4=h0b41bf4_0 + - reproc-cpp=14.2.4=hcb278e6_0 + - requests=2.31.0=pyhd8ed1ab_0 + - ruamel.yaml=0.17.32=py310h2372a71_0 + - ruamel.yaml.clib=0.2.7=py310h1fa729e_1 + - s2n=1.3.46=h06160fa_0 + - setuptools=68.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.1.10=h9fff704_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tblib=1.7.0=pyhd8ed1ab_0 + - tini=0.19.0=h166bdaf_1 + - tk=8.6.12=h27826a3_0 + - toolz=0.12.0=pyhd8ed1ab_0 + - tornado=6.3.2=py310h2372a71_0 + - tqdm=4.65.0=pyhd8ed1ab_1 + - traitlets=5.9.0=pyhd8ed1ab_0 + - typing-extensions=4.7.1=hd8ed1ab_0 + - typing_extensions=4.7.1=pyha770c72_0 + - tzdata=2023c=h71feb2d_0 + - ucx=1.14.1=hf587318_2 + - urllib3=2.0.3=pyhd8ed1ab_1 + - wcwidth=0.2.6=pyhd8ed1ab_0 + - wheel=0.40.0=pyhd8ed1ab_0 + - xarray=2023.7.0=pyhd8ed1ab_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yaml-cpp=0.7.0=h27087fc_2 + - yarl=1.9.2=py310h2372a71_0 + - zeromq=4.3.4=h9c3ff4c_1 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.16.2=pyhd8ed1ab_0 + - zstandard=0.19.0=py310h1275a96_2 + - zstd=1.5.2=hfc55251_7 +prefix: /opt/conda diff --git a/dockerfiles/minimal-notebook/notebooks/intro.ipynb b/dockerfiles/minimal-notebook/notebooks/intro.ipynb.tpl similarity index 94% rename from dockerfiles/minimal-notebook/notebooks/intro.ipynb rename to dockerfiles/minimal-notebook/notebooks/intro.ipynb.tpl index f79ae6f..7b5ae43 100644 --- a/dockerfiles/minimal-notebook/notebooks/intro.ipynb +++ b/dockerfiles/minimal-notebook/notebooks/intro.ipynb.tpl @@ -127,16 +127,29 @@ "```\n", "> To enable this feature, remove the `.disable-local-data-shortcut` file.\n", "### Why is there a `intro.ipynb` in my home directory?\n", - "> This is the `Nimbus` introduction guide. It's provided to keep you updated on changes and provide access to an FAQ.\n", - "# Changelog\n", - "- Initial commit\n", - "- Adds nbgitpuller" + "> This is the `Nimbus` introduction guide. It's provided to keep you updated on changes and provide access to an FAQ." ] }, { "cell_type": "code", "execution_count": null, "id": "13b09108-ab94-4d02-8c42-b74c6631b3c8", + "metadata": { + "jupyter": { + "source_hidden": true + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from IPython.display import Markdown, display\n", + "display(Markdown(\"/changelog.md\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2322f910-cd56-45db-909f-37457ac218f7", "metadata": {}, "outputs": [], "source": [] diff --git a/dockerfiles/minimal-notebook/scripts/install-envs.sh b/dockerfiles/minimal-notebook/scripts/install-envs.sh new file mode 100755 index 0000000..91c6008 --- /dev/null +++ b/dockerfiles/minimal-notebook/scripts/install-envs.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +mamba env update -n base -f /conda-envs/dask-gateway.yaml + +for env in $(find /conda-envs -type f -printf "%f\n"); do + name="${env%.*}" + mamba env create -n "${name}" -f "/conda-envs/${env}" + source /opt/conda/etc/profile.d/conda.sh + conda activate "${name}" + python -m ipykernel install --name="${name}" + conda deactivate +done diff --git a/dockerfiles/minimal-notebook/scripts/render-intro.sh b/dockerfiles/minimal-notebook/scripts/render-intro.sh new file mode 100644 index 0000000..b58ed3e --- /dev/null +++ b/dockerfiles/minimal-notebook/scripts/render-intro.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +jupyter nbconvert --execute --to notebook --stdout /notebooks/intro.ipynb.tpl > /notebooks/intro.ipynb From e7afb5897b221a26d1e471deb450c01f90ab299b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 1 Aug 2023 18:38:42 -0700 Subject: [PATCH 4/6] Adds changelog --- dockerfiles/minimal-notebook-gpu/Dockerfile | 4 ++++ dockerfiles/minimal-notebook-gpu/Makefile | 2 ++ 2 files changed, 6 insertions(+) diff --git a/dockerfiles/minimal-notebook-gpu/Dockerfile b/dockerfiles/minimal-notebook-gpu/Dockerfile index 87e409d..ac51ab5 100644 --- a/dockerfiles/minimal-notebook-gpu/Dockerfile +++ b/dockerfiles/minimal-notebook-gpu/Dockerfile @@ -9,4 +9,8 @@ RUN --mount=type=cache,target=/opt/conda/pkgs \ cupy && \ chown -R ${NB_UID}:${NB_GID} /home/jovyan +COPY changelog.md /changelog.md + +RUN source /scripts/render-intro.sh + USER $NB_USER diff --git a/dockerfiles/minimal-notebook-gpu/Makefile b/dockerfiles/minimal-notebook-gpu/Makefile index 6857017..7619d54 100644 --- a/dockerfiles/minimal-notebook-gpu/Makefile +++ b/dockerfiles/minimal-notebook-gpu/Makefile @@ -4,4 +4,6 @@ VERSION = 0.1.0 build: ARGS = --build-arg TAG=$(shell tbump -C ../minimal-notebook current-version) +run: ARGS = -p 8888:8888 + include ../Makefile From 0724eb30a4461d701699bffa55a8db33c6d81ebf Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 1 Aug 2023 18:41:28 -0700 Subject: [PATCH 5/6] Bump to 0.1.8 --- dockerfiles/minimal-notebook/Makefile | 2 +- dockerfiles/minimal-notebook/tbump.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/minimal-notebook/Makefile b/dockerfiles/minimal-notebook/Makefile index 6b0e265..15adc95 100644 --- a/dockerfiles/minimal-notebook/Makefile +++ b/dockerfiles/minimal-notebook/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL = build -VERSION = 0.1.7 +VERSION = 0.1.8 run: ARGS = -p 8888:8888 diff --git a/dockerfiles/minimal-notebook/tbump.toml b/dockerfiles/minimal-notebook/tbump.toml index e2fb4df..8b1ff20 100644 --- a/dockerfiles/minimal-notebook/tbump.toml +++ b/dockerfiles/minimal-notebook/tbump.toml @@ -1,5 +1,5 @@ [version] -current = "0.1.7" +current = "0.1.8" regex = ''' (?P\d+) From 62433c85f007d47992bdc6781c671127101352f1 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 1 Aug 2023 18:42:07 -0700 Subject: [PATCH 6/6] Bump to 0.1.1 --- dockerfiles/minimal-notebook-gpu/Makefile | 2 +- dockerfiles/minimal-notebook-gpu/tbump.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/minimal-notebook-gpu/Makefile b/dockerfiles/minimal-notebook-gpu/Makefile index 7619d54..040e2e9 100644 --- a/dockerfiles/minimal-notebook-gpu/Makefile +++ b/dockerfiles/minimal-notebook-gpu/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL = build -VERSION = 0.1.0 +VERSION = 0.1.1 build: ARGS = --build-arg TAG=$(shell tbump -C ../minimal-notebook current-version) diff --git a/dockerfiles/minimal-notebook-gpu/tbump.toml b/dockerfiles/minimal-notebook-gpu/tbump.toml index 05fa32b..c59caf4 100644 --- a/dockerfiles/minimal-notebook-gpu/tbump.toml +++ b/dockerfiles/minimal-notebook-gpu/tbump.toml @@ -1,5 +1,5 @@ [version] -current = "0.1.0" +current = "0.1.1" regex = ''' (?P\d+)