2024.4 base upgrade (nb7/py311), *packages upgraded

.github/workflows/docker.yml

@@ -3,6 +3,7 @@ on:
   schedule:
     - cron: '0 2 * * *'
   push:
+  workflow_dispatch:
 
 env:
   REGISTRY: ghcr.io

Dockerfile

@@ -1,36 +1,44 @@
-ARG BASE_CONTAINER=ucsdets/datahub-base-notebook:2023.2-stable
+ARG BASE_CONTAINER=ghcr.io/ucsd-ets/datascience-notebook:2024.4-stable
 FROM $BASE_CONTAINER
 
 LABEL maintainer="UC San Diego ITS/ETS <[email protected]>"
 
 USER root
 
-######################### ##
-# Requested for DSC170 WI23
+########################
+# Requested for DSC170 WI25
 COPY requirements.txt /home/jovyan
 
 RUN apt update -y && \
     apt-get install software-properties-common -y && \
     add-apt-repository universe && \
     apt update -y && \
     apt install graphviz -y
-RUN mamba install -c esri arcgis=2.2.0 numpy=1.26.2 -y
 
-RUN pip install --upgrade pip setuptools wheel
-RUN pip install --upgrade nbconvert
-
-RUN mamba install -c conda-forge geopandas cartopy pygeos pysal contextily osmnx jupyterlab_widgets -y
+RUN jupyter labextension install @dafeliton/jupyterlab-notebookparams
 
+# Install geospatial packages first
 RUN pip uninstall pillow fiona -y && \
-  pip install -r ~/requirements.txt && \
-	pip install --upgrade fiona
+    pip install -r ~/requirements.txt && \
+    pip install --upgrade fiona
+
+# pygeos is deprecated and has been merged with shapely. (pygeos works around python GIL)
+# https://github.com/shapely/shapely
+
+# geopandas/cartopy/etc. error when importing sqlite3 from python:
+# ImportError: /opt/conda/lib/python3.11/lib-dynload/_sqlite3.cpython-311-x86_64-linux-gnu.so: undefined symbol: sqlite3_deserialize 
+# Moved to requirements.txt
+
+RUN mamba install -c esri arcgis arcgis-mapping -y && \
+    fix-permissions $CONDA_DIR && \
+    fix-permissions /home/$NB_USER && \
+    mamba clean --all
+
+RUN pip install --upgrade pip setuptools wheel
+RUN pip install --upgrade nbconvert
 
-RUN jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
-	jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
-	jupyter labextension install [email protected]
 
-RUN jupyter nbextension enable  --py --sys-prefix arcgis && \
-	jupyter nbextension enable --py --sys-prefix arcgis
+# RUN pip install "numpy<2"
 
 USER $NB_UID
 
@@ -39,3 +47,4 @@ RUN rm -rf /home/jovyan/requirements.txt
 
 ENV USE_PYGEOS=0
 
+RUN pip install numpy==1.26.4

README.md

@@ -3,14 +3,3 @@
 DataHub notebook for GPEC 447 - Data Science/Spatial Analysis
 
 Sample notebook arcgis_test.ipynb is copied into container at /opt
-
-# Usage
-
-## Map Widget
-
-Enable the map widget by running these 2 commands in a New -> Terminal:
-
-```
-jupyter nbextension enable --py --user widgetsnbextension
-jupyter nbextension enable --py --user arcgis
-```

notes.md

@@ -0,0 +1,38 @@
+### 12/1/24 Notes:
+
+1. Installing certain python packages breaks our python installation (failed to import sqlite3, prevents JLab startup and breaks the image)
+
+Solution: Install them using pip. A more permanent solution can be found but at this time only a certain number of python packages seem to trigger this issue. 
+
+2. IPython.notebook.* no longer exists in JupyterLab/NB7+.
+
+Solution: There is no 1:1 drop-in alternative to this as far as I'm aware. However, here are a few potential replacements:
+
+- https://gist.github.com/gnestor/f1893e0226ced227e910f11b769adc06
+
+- The ```pylab``` extension: https://stackoverflow.com/questions/68585590/run-all-below-action-in-jupyterlab-programatically/71944193#71944193
+
+- Using the following snippet is a decent start:
+
+```
+%%javascript
+    console.log('hello')
+```
+
+This logs "hello" to the developer console.
+
+```
+from IPython.display import Javascript, display
+from ipywidgets import HTML
+
+html_widget = HTML()
+display(html_widget)
+
+# Use Javascript to set the innerHTML of the widget
+display(Javascript("""
+    var test_variable = 'Hello from Javascript!';
+    element.innerHTML = test_variable;
+"""))
+```
+
+This displays an HTML widget to get similar behavior to the original ```IPython.notebook.kernel.execute()``` method.

requirements.txt

@@ -1,18 +1,41 @@
-pillow==9.1
-pyproj==3.3 
+pillow
+pyproj
 openpyxl 
 cenpy 
 python-Levenshtein 
 geopy 
-ipyleaflet 
-ipywidgets 
-keplergl 
+ipyleaflet
+keplergl
 mesa-geo 
 mesa 
 matplotlib 
-jupyterlab==2.3.1 
-shapely==2.0.0 
-pyshp==2.3.1 
-pandas==1.5.2 
+shapely
+pyshp
+pandas
 censusdata 
 graphviz
+geopandas
+cartopy
+pysal
+contextily
+jupyterlab_widgets
+osmnx
+numpy==1.26.4
+
+# TODO: keplergl/bokeh conflict with their dependency on ipywidgets as of 12/5/24. keplergl wants ipyiwdgets<8 >=7.8.1, but bokeh wants ==8.*.
+# May have to pick one or the other
+
+# New packages
+ipywidgets
+panel
+statsmodels
+pyvis
+networkx
+holoviews
+hvplot
+geopandas
+bokeh
+jupyter_bokeh
+jupyterlab-open-url-parameter
+pygris
+shap