From 560119bded97e825c08ad56c570da85f0e81672b Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:23:04 -0700 Subject: [PATCH 01/34] test path change just in case then test apt (but this may be tricky since we may still need to mess with paths. ubuntu doesn't provide 3.3.2 except in their pre-release repo for the next major dist) --- images/rstudio-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index db504d2f..ed0c5c50 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -33,7 +33,7 @@ RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPE ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ SHELL=/bin/bash \ - LD_LIBRARY_PATH="/usr/lib/R/lib:/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server:/opt/conda/lib/R/lib" + LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server:/opt/conda/lib/R/lib" COPY ./lib /opt/k8s-support/lib From 8554c9fb33cc03286302feb564c393f2e1b66664 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 04:02:46 -0700 Subject: [PATCH 02/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index ed0c5c50..92cdad63 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -32,8 +32,8 @@ RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPE ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash \ - LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server:/opt/conda/lib/R/lib" + SHELL=/bin/bash + #LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server:/opt/conda/lib/R/lib" COPY ./lib /opt/k8s-support/lib From dac21307139e8ae38804c81bc281ce7d944523fb Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 04:40:25 -0700 Subject: [PATCH 03/34] try to fix curl rather than openssl --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 92cdad63..b10ac011 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -32,8 +32,8 @@ RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPE ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash - #LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server:/opt/conda/lib/R/lib" + SHELL=/bin/bash \ + LD_LIBRARY_PATH="/usr/lib:/usr/lib/x86_64-linux-gnu:/opt/conda/lib/R/lib" COPY ./lib /opt/k8s-support/lib From f77d43df0f1a0ca6ec0e035e1e0eceae0083fbdf Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 05:27:54 -0700 Subject: [PATCH 04/34] curl renviron --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index b10ac011..d3d8e98a 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -28,12 +28,12 @@ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed -RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site +RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap'; echo 'CURL=/usr/bin/curl') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ SHELL=/bin/bash \ - LD_LIBRARY_PATH="/usr/lib:/usr/lib/x86_64-linux-gnu:/opt/conda/lib/R/lib" + LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/opt/conda/lib/R/lib" COPY ./lib /opt/k8s-support/lib From 0eb5d8791a5e42b2e41cb7f8b628fa77bb07a27b Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 05:34:05 -0700 Subject: [PATCH 05/34] path var --- images/rstudio-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index d3d8e98a..a9a5a516 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -28,7 +28,7 @@ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed -RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap'; echo 'CURL=/usr/bin/curl') >> /opt/conda/lib/R/etc/Renviron.site +RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap'; echo 'PATH=/usr/bin:$PATH') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ From 88caa72e616a2afb4d15db8b5fd369c18b40c208 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 05:51:14 -0700 Subject: [PATCH 06/34] https://github.com/rstudio/rstudio/issues/14060 --- images/rstudio-notebook/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index a9a5a516..e6bd1e7d 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -25,10 +25,12 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy -RUN mkdir -p /etc/rstudio && echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf +RUN mkdir -p /etc/rstudio && \ + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ + echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed -RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap'; echo 'PATH=/usr/bin:$PATH') >> /opt/conda/lib/R/etc/Renviron.site +RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ From 59e438b485738dbf9af0f43f065158e1d4d786be Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:58:44 -0700 Subject: [PATCH 07/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index e6bd1e7d..06272b4f 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -34,8 +34,8 @@ RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPE ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash \ - LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/opt/conda/lib/R/lib" + SHELL=/bin/bash + #LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/opt/conda/lib/R/lib" COPY ./lib /opt/k8s-support/lib From f7098d669c348565726d79513a604fbef8eebcbf Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:48:27 -0700 Subject: [PATCH 08/34] test ld-library-blank --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 06272b4f..0bc45e2c 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -34,8 +34,8 @@ RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPE ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash - #LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/opt/conda/lib/R/lib" + SHELL=/bin/bash \ + LD_LIBRARY_PATH="" COPY ./lib /opt/k8s-support/lib From 65566d40bcc2cc6ec6edf0e3815a7b049d025ba7 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:13:07 -0700 Subject: [PATCH 09/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 0bc45e2c..e99d584e 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -27,7 +27,7 @@ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ - echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf + echo 'rsession-ld-library-path=/usr/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site From 01a533ee9eef8c4d9c08f83fb0ccdf5325a36dcd Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:19:30 -0700 Subject: [PATCH 10/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index e99d584e..4c8533f1 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -17,7 +17,7 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ wget ${RSTUDIO_URL} && \ gdebi -n ${RSTUDIO_PKG} && \ rm -f ${RSTUDIO_PKG} && \ - echo '/opt/conda/lib/R/lib' > /etc/ld.so.conf.d/r.conf && /sbin/ldconfig -v && \ + #echo '/opt/conda/lib/R/lib' > /etc/ld.so.conf.d/r.conf && /sbin/ldconfig -v && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ rm -f /usr/bin/R && \ chmod -R g=u /var/lib/rstudio-server && \ @@ -26,8 +26,8 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ - echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ - echo 'rsession-ld-library-path=/usr/lib' >> /etc/rstudio/rserver.conf + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf + #echo 'rsession-ld-library-path=/usr/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site From 623c38f908e7cfb0b44bf7cfd317e3cc1ad83e79 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:47:42 -0700 Subject: [PATCH 11/34] ssh-keygen fixes pending --- images/rstudio-notebook/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 4c8533f1..92d18248 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -17,7 +17,7 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ wget ${RSTUDIO_URL} && \ gdebi -n ${RSTUDIO_PKG} && \ rm -f ${RSTUDIO_PKG} && \ - #echo '/opt/conda/lib/R/lib' > /etc/ld.so.conf.d/r.conf && /sbin/ldconfig -v && \ + echo '/opt/conda/lib/R/lib' > /etc/ld.so.conf.d/r.conf && /sbin/ldconfig -v && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ rm -f /usr/bin/R && \ chmod -R g=u /var/lib/rstudio-server && \ @@ -27,15 +27,14 @@ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf - #echo 'rsession-ld-library-path=/usr/lib' >> /etc/rstudio/rserver.conf + echo "rsession-ld-library-path=/opt/conda/lib" >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash \ - LD_LIBRARY_PATH="" + SHELL=/bin/bash COPY ./lib /opt/k8s-support/lib From e1d50e91563739b056a814965529f2ee78f697a2 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:54:43 -0700 Subject: [PATCH 12/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 92d18248..6c789a97 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -26,8 +26,8 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ - echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf - echo "rsession-ld-library-path=/opt/conda/lib" >> /etc/rstudio/rserver.conf + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ + echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site From df7e8ea22c3ec02c8d8a5aee3a784d7293a347f6 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Tue, 8 Oct 2024 23:40:00 -0700 Subject: [PATCH 13/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 6c789a97..7023d8d5 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -26,8 +26,8 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ - echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ - echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf + #echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site From e85fa9c7bee14bc61fef81e3e7956ab31da5670a Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 00:58:34 -0700 Subject: [PATCH 14/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 7023d8d5..6c789a97 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -26,8 +26,8 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ - echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf - #echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ + echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site From e4495957c08d8f2378b7a67f4a8e7f28d39d06e4 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:14:33 -0700 Subject: [PATCH 15/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 6c789a97..c14983d7 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -26,11 +26,11 @@ RUN ln -s /opt/conda/bin/R /usr/bin/R && \ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ - echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf && \ - echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf + echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf + #echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed -RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site +RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so:/lib/x86_64-linux-gnu/libcurl-gnutls.so.3'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ From 9b7e5b8201b58985f024b1cc0887e637268d3fdb Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:27:30 -0700 Subject: [PATCH 16/34] more tests --- images/rstudio-notebook/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index c14983d7..51f129ad 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -27,14 +27,14 @@ RUN pip install jupyter-rsession-proxy RUN mkdir -p /etc/rstudio && \ echo 'auth-minimum-user-id=100' >> /etc/rstudio/rserver.conf - #echo 'rsession-ld-library-path=/opt/conda/lib' >> /etc/rstudio/rserver.conf ## Shim in our nss-wrapper so that user getpwent() lookups succeed -RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so:/lib/x86_64-linux-gnu/libcurl-gnutls.so.3'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site +RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPER_PASSWD=/tmp/passwd.wrap'; echo 'NSS_WRAPPER_GROUP=/tmp/group.wrap') >> /opt/conda/lib/R/etc/Renviron.site ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ - SHELL=/bin/bash + SHELL=/bin/bash \ + LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/opt/conda/lib" COPY ./lib /opt/k8s-support/lib From ae98d4e3d948c7cfc37ffe0fc0270a884ad27ad8 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:53:31 -0700 Subject: [PATCH 17/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 06272b4f..340d2953 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -10,6 +10,8 @@ USER root ENV RSTUDIO_PKG=rstudio-server-2024.04.2-764-amd64.deb ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/${RSTUDIO_PKG} +RUN mamba install openssl=3.0.2 + ## rstudio installation expects R to live in /usr/bin, /bin/, etc. RUN ln -s /opt/conda/bin/R /usr/bin/R && \ apt-get update && \ From c9f56451a6781ea61223573272263ce876075f96 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:10:26 -0700 Subject: [PATCH 18/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 340d2953..06272b4f 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -10,8 +10,6 @@ USER root ENV RSTUDIO_PKG=rstudio-server-2024.04.2-764-amd64.deb ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/${RSTUDIO_PKG} -RUN mamba install openssl=3.0.2 - ## rstudio installation expects R to live in /usr/bin, /bin/, etc. RUN ln -s /opt/conda/bin/R /usr/bin/R && \ apt-get update && \ From df7b5aaf01807aa0a69fc88b95c0c1331cb39309 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:10:58 -0700 Subject: [PATCH 19/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 51f129ad..356d454d 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -10,6 +10,8 @@ USER root ENV RSTUDIO_PKG=rstudio-server-2024.04.2-764-amd64.deb ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/${RSTUDIO_PKG} +RUN mamba install openssl=3.0.2 + ## rstudio installation expects R to live in /usr/bin, /bin/, etc. RUN ln -s /opt/conda/bin/R /usr/bin/R && \ apt-get update && \ From ffdf2371467ba368039e6dbd3c7253db3d2f6ae8 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:21:31 -0700 Subject: [PATCH 20/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 356d454d..87306f0e 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -10,8 +10,6 @@ USER root ENV RSTUDIO_PKG=rstudio-server-2024.04.2-764-amd64.deb ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/${RSTUDIO_PKG} -RUN mamba install openssl=3.0.2 - ## rstudio installation expects R to live in /usr/bin, /bin/, etc. RUN ln -s /opt/conda/bin/R /usr/bin/R && \ apt-get update && \ @@ -36,7 +34,7 @@ RUN ( echo 'LD_PRELOAD=/opt/k8s-support/lib/libnss_wrapper.so'; echo 'NSS_WRAPPE ## The desktop package uses /usr/lib/rstudio/bin ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ SHELL=/bin/bash \ - LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/opt/conda/lib" + LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu" COPY ./lib /opt/k8s-support/lib From f0640d62a14f163e7b9cd71efb32f70f4b63b587 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:37:42 -0700 Subject: [PATCH 21/34] workaround ssh-keygen --- images/rstudio-notebook/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 87306f0e..29b82704 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -36,6 +36,15 @@ ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ SHELL=/bin/bash \ LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu" +# Move the original ssh-keygen binary +RUN mv /usr/bin/ssh-keygen /usr/bin/ssh-keygen.orig + +# Create a wrapper script for ssh-keygen +RUN echo '#!/bin/bash' > /usr/bin/ssh-keygen && \ + echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh-keygen && \ + echo 'exec /usr/bin/ssh-keygen.orig "$@"' >> /usr/bin/ssh-keygen && \ + chmod +x /usr/bin/ssh-keygen + COPY ./lib /opt/k8s-support/lib COPY ./test ./integration_tests /home/jovyan/ From c0596c114c01a0cc19817c808c85b377ebda7721 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:16:39 -0700 Subject: [PATCH 22/34] Wrap git exec --- images/rstudio-notebook/Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 29b82704..31ed64d1 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -36,15 +36,21 @@ ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ SHELL=/bin/bash \ LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu" -# Move the original ssh-keygen binary +### OpenSSL Conda Workarounds +# Move the original binary RUN mv /usr/bin/ssh-keygen /usr/bin/ssh-keygen.orig - -# Create a wrapper script for ssh-keygen +# Create a wrapper script RUN echo '#!/bin/bash' > /usr/bin/ssh-keygen && \ echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh-keygen && \ echo 'exec /usr/bin/ssh-keygen.orig "$@"' >> /usr/bin/ssh-keygen && \ chmod +x /usr/bin/ssh-keygen +RUN mv /usr/bin/git /usr/bin/git.org +RUN echo '#!/bin/bash' > /usr/bin/git && \ + echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/git && \ + echo 'exec /usr/bin/git.orig "$@"' >> /usr/bin/git && \ + chmod +x /usr/bin/git + COPY ./lib /opt/k8s-support/lib COPY ./test ./integration_tests /home/jovyan/ From b497f8a5343865fdce5f069671267e30a70470b0 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:17:37 -0700 Subject: [PATCH 23/34] minor spelling mistake --- images/rstudio-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 31ed64d1..e4582649 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -45,7 +45,7 @@ RUN echo '#!/bin/bash' > /usr/bin/ssh-keygen && \ echo 'exec /usr/bin/ssh-keygen.orig "$@"' >> /usr/bin/ssh-keygen && \ chmod +x /usr/bin/ssh-keygen -RUN mv /usr/bin/git /usr/bin/git.org +RUN mv /usr/bin/git /usr/bin/git.orig RUN echo '#!/bin/bash' > /usr/bin/git && \ echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/git && \ echo 'exec /usr/bin/git.orig "$@"' >> /usr/bin/git && \ From 1b2cba984d6e2339c9dc0c39ddd9207706ad8ff0 Mon Sep 17 00:00:00 2001 From: dafeliton Date: Sat, 12 Oct 2024 11:19:41 -0700 Subject: [PATCH 24/34] add ssh exec --- images/rstudio-notebook/Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index e4582649..3d7ef8ad 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -37,20 +37,27 @@ ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu" ### OpenSSL Conda Workarounds -# Move the original binary +# ssh-keygen RUN mv /usr/bin/ssh-keygen /usr/bin/ssh-keygen.orig -# Create a wrapper script RUN echo '#!/bin/bash' > /usr/bin/ssh-keygen && \ echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh-keygen && \ echo 'exec /usr/bin/ssh-keygen.orig "$@"' >> /usr/bin/ssh-keygen && \ chmod +x /usr/bin/ssh-keygen +# git RUN mv /usr/bin/git /usr/bin/git.orig RUN echo '#!/bin/bash' > /usr/bin/git && \ echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/git && \ echo 'exec /usr/bin/git.orig "$@"' >> /usr/bin/git && \ chmod +x /usr/bin/git +# ssh +RUN mv /usr/bin/ssh /usr/bin/ssh.orig +RUN echo '#!/bin/bash' > /usr/bin/ssh && \ + echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh && \ + echo 'exec /usr/bin/git.orig "$@"' >> /usr/bin/ssh && \ + chmod +x /usr/bin/ssh + COPY ./lib /opt/k8s-support/lib COPY ./test ./integration_tests /home/jovyan/ From eae2073d8761eaf8fb8453760d93f546d21da4a2 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Sat, 12 Oct 2024 23:44:48 -0700 Subject: [PATCH 25/34] Update Dockerfile --- images/rstudio-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 3d7ef8ad..537fad4c 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -55,7 +55,7 @@ RUN echo '#!/bin/bash' > /usr/bin/git && \ RUN mv /usr/bin/ssh /usr/bin/ssh.orig RUN echo '#!/bin/bash' > /usr/bin/ssh && \ echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh && \ - echo 'exec /usr/bin/git.orig "$@"' >> /usr/bin/ssh && \ + echo 'exec /usr/bin/ssh.orig "$@"' >> /usr/bin/ssh && \ chmod +x /usr/bin/ssh COPY ./lib /opt/k8s-support/lib From dea5d7e6bd2e40fdc3c3285b642a2cd6e21b6d4b Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Sun, 13 Oct 2024 00:13:12 -0700 Subject: [PATCH 26/34] add loop for mapped execs --- images/rstudio-notebook/Dockerfile | 33 ++++++++++++------------------ 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 537fad4c..82205aa9 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -37,26 +37,19 @@ ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu" ### OpenSSL Conda Workarounds -# ssh-keygen -RUN mv /usr/bin/ssh-keygen /usr/bin/ssh-keygen.orig -RUN echo '#!/bin/bash' > /usr/bin/ssh-keygen && \ - echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh-keygen && \ - echo 'exec /usr/bin/ssh-keygen.orig "$@"' >> /usr/bin/ssh-keygen && \ - chmod +x /usr/bin/ssh-keygen - -# git -RUN mv /usr/bin/git /usr/bin/git.orig -RUN echo '#!/bin/bash' > /usr/bin/git && \ - echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/git && \ - echo 'exec /usr/bin/git.orig "$@"' >> /usr/bin/git && \ - chmod +x /usr/bin/git - -# ssh -RUN mv /usr/bin/ssh /usr/bin/ssh.orig -RUN echo '#!/bin/bash' > /usr/bin/ssh && \ - echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/ssh && \ - echo 'exec /usr/bin/ssh.orig "$@"' >> /usr/bin/ssh && \ - chmod +x /usr/bin/ssh +# List of binaries you want to wrap +ENV FORCED_BINARIES="ssh ssh-add ssh-agent ssh-copy-id ssh-keygen ssh-keyscan sshd git" + +# Loop over each binary and apply the wrapper +RUN for bin in $FORCED_BINARIES; do \ + if [ -f /usr/bin/$bin ]; then \ + mv /usr/bin/$bin /usr/bin/$bin.orig && \ + echo '#!/bin/bash' > /usr/bin/$bin && \ + echo 'export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu' >> /usr/bin/$bin && \ + echo 'exec /usr/bin/'$bin'.orig "$@"' >> /usr/bin/$bin && \ + chmod +x /usr/bin/$bin; \ + fi; \ + done COPY ./lib /opt/k8s-support/lib From 61c9e3cb6f70554b99cf9ce0eb02dea7fcfb47b4 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Mon, 14 Oct 2024 00:58:07 -0700 Subject: [PATCH 27/34] Update tag.yml --- .github/workflows/tag.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index b84b76dc..cad9974d 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -11,6 +11,10 @@ on: type: boolean description: 'dry-run: retrieve original image names ONLY' required: true + skipGPUTest: + type: boolean + description: 'skip-gpu-test: Skips testing via the GPU. Never use this unless dsmlp-login is encountering issues!' + required: true jobs: @@ -43,6 +47,7 @@ jobs: with: originalTag: ${{ github.event.inputs.originalTag }} dryRun: ${{ github.event.inputs.dryRun == 'true' }} + skipGPUTest: ${{ github.event.inputs.skipGPUTest == 'false' }} secrets: inherit From 72d5cbfe859e863bc46a828fec74af2fcfb71f88 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Mon, 14 Oct 2024 01:01:50 -0700 Subject: [PATCH 28/34] Update tag.yml --- .github/workflows/tag.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index cad9974d..6cd14e11 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -44,10 +44,10 @@ jobs: gpu_test: uses: ./.github/workflows/test_gpu.yml + if: ${{ github.event.inputs.skipGPUTest == 'false' }} with: originalTag: ${{ github.event.inputs.originalTag }} dryRun: ${{ github.event.inputs.dryRun == 'true' }} - skipGPUTest: ${{ github.event.inputs.skipGPUTest == 'false' }} secrets: inherit From 2cd5288927c4da2b36e9217b79131ec417e1ab65 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Mon, 14 Oct 2024 01:06:53 -0700 Subject: [PATCH 29/34] Update tag.yml --- .github/workflows/tag.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 6cd14e11..57c30ef2 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -52,9 +52,8 @@ jobs: inherit tagging: - needs: - - gpu_test - - start-runner + needs: start-runner + if: ${{ github.event.inputs.skipGPUTest == 'true' || needs.gpu_test.result == 'success' }} runs-on: ${{ needs.start-runner.outputs.label }} steps: - uses: actions/checkout@v3 From f908ede3258ff6ff64a734dbe7909cd4895abdeb Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Mon, 14 Oct 2024 01:11:22 -0700 Subject: [PATCH 30/34] Update test_gpu.yml --- .github/workflows/test_gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_gpu.yml b/.github/workflows/test_gpu.yml index b3aa8c70..f75ca337 100644 --- a/.github/workflows/test_gpu.yml +++ b/.github/workflows/test_gpu.yml @@ -59,7 +59,7 @@ jobs: echo "About to SSH." - output=$(sshpass -p ${{ secrets.DSMLP_LOGIN_PASSWORD }} ssh grader-test-01@dsmlp-login.ucsd.edu '/opt/launch-sh/bin/launch.sh -n 22 -i ghcr.io/ucsd-ets/scipy-ml-notebook:${{ github.event.inputs.originalTag }} -P Always -g 1 -c 4 -m 32 -f pytest /opt/workflow_tests') + output=$(sshpass -p ${{ secrets.DSMLP_LOGIN_PASSWORD }} ssh grader-test-01@dsmlp-login.ucsd.edu '/opt/launch-sh/bin/launch.sh -i ghcr.io/ucsd-ets/scipy-ml-notebook:${{ github.event.inputs.originalTag }} -P Always -g 1 -c 4 -m 32 -f pytest /opt/workflow_tests') echo "$output" if [[ "$output" == *"FAILURES"* ]]; then From 57fb8ab9b806f6c7f78c78ade2217b0631959311 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <127802411+dafeliton@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:19:52 -0700 Subject: [PATCH 31/34] Document rstudio madness --- images/rstudio-notebook/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index 82205aa9..eabb627d 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -37,6 +37,15 @@ ENV PATH="${PATH}:/usr/lib/rstudio-server/bin" \ LD_LIBRARY_PATH="/opt/conda/lib:/usr/lib/x86_64-linux-gnu" ### OpenSSL Conda Workarounds +# When we upgraded to Python3.11/JLab 4 upstreamed from Jupyter, our distro was also upgraded from Ubuntu 20 to 22. +# This introduced a new issue where RStudio would prefer /opt/conda/lib's OpenSSL instead of /usr/lib when trying to download packages using internet.so (internal RStudio lib). +# https://github.com/rstudio/rstudio/issues/14060 - the fix described was setting LD_LIBRARY_PATH to /opt/conda/lib:/usr/lib/x86_64-linux-gnu (globally or just for RStudio). +# However, this fix is problematic because many executables such as SSH*/git rely on the OpenSSL version being installed by apt (since they are installed by apt). +# Post-fix, when you tried to launch ssh/git in the RStudio Termina, you would receieve an "OpenSSL mismatch" error. +# To fix this, we simply added a bash wrapper for all affected binaries that forces them to always use the correct LD_LIBRARY_PATH (apt). +# If another binary encounters a "OpenSSL mismatch error," add it here. +# Hopefully a better fix can be found - the core issue here seems to be conda and its obtrusiveness + # List of binaries you want to wrap ENV FORCED_BINARIES="ssh ssh-add ssh-agent ssh-copy-id ssh-keygen ssh-keyscan sshd git" From e49139a3fb5b7e255345c53046bb545733005edf Mon Sep 17 00:00:00 2001 From: RockfordMankiniUCSD <92760584+RockfordMankiniUCSD@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:12:12 -0800 Subject: [PATCH 32/34] Update tag_global_stable.yml --- .github/workflows/tag_global_stable.yml | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/tag_global_stable.yml b/.github/workflows/tag_global_stable.yml index 394ae3e9..23f1ae2c 100644 --- a/.github/workflows/tag_global_stable.yml +++ b/.github/workflows/tag_global_stable.yml @@ -24,6 +24,35 @@ jobs: docker image prune -a -f docker container prune -f + - name: Free disk space + run: | + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + sudo rm -rf \ + /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /usr/lib/jvm || true + echo "some directories deleted" + sudo apt install aptitude -y >/dev/null 2>&1 + sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ + esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ + google-cloud-sdk imagemagick \ + libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ + mercurial apt-transport-https mono-complete libmysqlclient \ + unixodbc-dev yarn chrpath libssl-dev libxft-dev \ + libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ + snmp pollinate libpq-dev postgresql-client powershell ruby-full \ + sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ + -y -f >/dev/null 2>&1 + sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 + sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 + sudo apt-get autoremove -y >/dev/null 2>&1 + sudo apt-get autoclean -y >/dev/null 2>&1 + echo "some packages purged" + - name: Set up Python 3.8 uses: actions/setup-python@v3 with: From 181f59bcee1bcba82362c06d622ff890eea6080c Mon Sep 17 00:00:00 2001 From: Rockford Mankini Date: Wed, 4 Dec 2024 13:40:37 -0800 Subject: [PATCH 33/34] attempt to build 2025.1 --- images/datascience-notebook/Dockerfile | 2 +- images/rstudio-notebook/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile index 278cf1d6..f172251e 100644 --- a/images/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -65,7 +65,7 @@ USER jovyan # Python/Mamba Deps ## Package versions ARG JUPYTERSERVER_VERSION=2.14.2 NBGRADER_VERSION=0.9.3 JUPYTERLAB_VERSION=4.2.4 NBCONVERT_VERSION=7.16.4 NOTEBOOK_VERSION=7.2.1 NBCLASSIC_VERSION=1.1.0 -ARG PANDAS_VERSION=2.2.2 STATSMODELS_VERSION=0.14.2 BOTTLENECK_VERSION=1.3.6 NUMEXPR_VERSION=2.8.4 +ARG PANDAS_VERSION=2.2.3 STATSMODELS_VERSION=0.14.4 BOTTLENECK_VERSION=1.4.2 NUMEXPR_VERSION=2.10.2 # Install essential+datascience pip packages ## mistune added for nbgrader issues diff --git a/images/rstudio-notebook/Dockerfile b/images/rstudio-notebook/Dockerfile index eabb627d..c0a41edc 100644 --- a/images/rstudio-notebook/Dockerfile +++ b/images/rstudio-notebook/Dockerfile @@ -7,7 +7,7 @@ USER root # Ubuntu 22 setup with v 2024.04.2-764 ## Follow instructions at https://www.rstudio.com/products/rstudio/download-server/ -ENV RSTUDIO_PKG=rstudio-server-2024.04.2-764-amd64.deb +ENV RSTUDIO_PKG=rstudio-server-2024.09.1-394-amd64.deb ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/${RSTUDIO_PKG} ## rstudio installation expects R to live in /usr/bin, /bin/, etc. From d8353a3d9c41ad3cdeb078bdbd6bf092d1f91b43 Mon Sep 17 00:00:00 2001 From: Rockford Mankini Date: Wed, 4 Dec 2024 14:13:53 -0800 Subject: [PATCH 34/34] update spec yml file for proper image name --- images/spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/spec.yml b/images/spec.yml index a61ff5b2..c6987ce6 100644 --- a/images/spec.yml +++ b/images/spec.yml @@ -25,7 +25,7 @@ images: #prepull: false #-- uncomment to disable prepulling behavior for scipy-ml. gives you space on machine in exchange for build time. tag: - prefix: "2024.4" + prefix: "2025.1" all_info_cmds: PY_VER: