Merge branch 'master' into ID-353-release-new-terra-notebook-utils

# Conflicts: # config/conf.json # terra-jupyter-aou/CHANGELOG.md # terra-jupyter-aou/Dockerfile # terra-jupyter-base/CHANGELOG.md # terra-jupyter-bioconductor/CHANGELOG.md # terra-jupyter-bioconductor/Dockerfile # terra-jupyter-gatk-ovtf/CHANGELOG.md # terra-jupyter-gatk-ovtf/Dockerfile # terra-jupyter-gatk/Dockerfile # terra-jupyter-hail/CHANGELOG.md # terra-jupyter-hail/Dockerfile # terra-jupyter-python/CHANGELOG.md # terra-jupyter-python/Dockerfile # terra-jupyter-r/CHANGELOG.md # terra-jupyter-r/Dockerfile
DataBiosphere · Jul 5, 2023 · fb9e444 · fb9e444
2 parents 20d2fcc + aeedecb
commit fb9e444
Show file tree

Hide file tree

Showing 50 changed files with 255 additions and 976 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -11,11 +11,11 @@ on:
           - terra-jupyter-base
           - terra-jupyter-bioconductor
           - terra-jupyter-gatk
-          - terra-jupyter-gatk-ovtf
           - terra-jupyter-hail
           - terra-jupyter-python
           - terra-jupyter-r
           - terra-rstudio-aou
+          - wondershaper
         default: terra-rstudio-rstudio
       branch:
         description: 'Branch to run the workflow on'
@@ -33,7 +33,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: '3.10'
 
       - id: auth
         uses: google-github-actions/auth@v1

diff --git a/.github/workflows/test-terra-jupyter-aou.yml b/.github/workflows/test-terra-jupyter-aou.yml
@@ -47,7 +47,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.10'
 
     - name: Free up some disk space
       run: sudo rm -rf /usr/share/dotnet

diff --git a/.github/workflows/test-terra-jupyter-base.yml b/.github/workflows/test-terra-jupyter-base.yml
@@ -47,7 +47,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.10'
 
     - name: Free up some disk space
       run: sudo rm -rf /usr/share/dotnet

diff --git a/.github/workflows/test-terra-jupyter-bioconductor.yml b/.github/workflows/test-terra-jupyter-bioconductor.yml
@@ -47,7 +47,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.10'
 
     - name: Free up some disk space
       run: sudo rm -rf /usr/share/dotnet

diff --git a/.github/workflows/test-terra-jupyter-gatk-ovtf.yml b/.github/workflows/test-terra-jupyter-gatk-ovtf.yml
diff --git a/.github/workflows/test-terra-jupyter-gatk.yml b/.github/workflows/test-terra-jupyter-gatk.yml
@@ -47,7 +47,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.10'
 
     - name: Free up some disk space
       run: sudo rm -rf /usr/share/dotnet

diff --git a/.github/workflows/test-terra-jupyter-hail.yml b/.github/workflows/test-terra-jupyter-hail.yml
@@ -47,7 +47,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.10'
 
     - name: Free up some disk space
       run: sudo rm -rf /usr/share/dotnet

diff --git a/.github/workflows/test-terra-jupyter-python.yml b/.github/workflows/test-terra-jupyter-python.yml
@@ -49,7 +49,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: '3.10'
 
     - name: Free up some disk space
       run: sudo rm -rf /usr/share/dotnet    
@@ -112,10 +112,8 @@ jobs:
           --workdir=/tests \
           --entrypoint="" \
           terra-jupyter-python:smoke-test \
-          /bin/sh -c "pip3 install --no-deps pytest iniconfig pluggy py; pytest"
-          # Use --no-deps to avoid an unneccessary upgrade of pyparsing which then causes tensorflow tests to fail.
-          # TODO: whenever we no longer need to pin pyparsing, simplify the above command to 'pip3 install pytest ; pytest'
-
+          /bin/sh -c "pip3 install pytest ; pytest"
+          
     - name: Test Python code specific to notebooks with nbconvert
       # Simply 'Cell -> Run All` these notebooks and expect no errors in the case of a successful run of the test suite.
       # If the tests throw any exceptions, execution of the notebooks will halt at that point. Look at the workflow

diff --git a/.gitignore b/.gitignore
@@ -15,4 +15,6 @@ package-lock.json
 .bash_history
 .conda/
 .python_history
-.keras/
+.keras/
+.ammonite/
+.metals/
diff --git a/README.md b/README.md
@@ -80,13 +80,19 @@ Detailed documentation on how to integrate the terra-docker image with Leonardo
 - Ensure that the `terra-docker-versions-candidate.json` file (which is what the UI sources the dropdown from) in the `terra-docjker-image-documentation-[env]` bucket correclty references your new docker image
 - [Update the terra-docker version candidate json](https://broadworkbench.atlassian.net/wiki/spaces/IA/pages/2519564289/Integrating+new+Terra+docker+images+with+Leonardo#6.-Update-terra-docker-versions-candidate.json)
 
+## Testing your image manually
 
+Build the image:
+run `docker build [your_dir] -t [name]`.
 
-If you wish to build locally, run `docker build [your_dir] -t [name]`. 
+`docker build terra-jupyter-base -t terra-jupyter-base`
 
-It is not advised to run build.sh locally, as this will push to the remote docker repo and delete the image locally upon completion.  
+If you're on an M1 and building an image from a locally built image, replace the current FROM command:
+
+`FROM --platform=linux/amd64 terra-jupyter-base`
+
+It is not advised to run build.sh locally, as this will push to the remote docker repo and delete the image locally upon completion. 
 
-## Testing your image manually
 All images can be run locally. For example:
 ```
 docker run --rm -it -p 8000:8000 us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:0.0.7
@@ -103,7 +109,7 @@ Running locally is conventient for quick development and exploring the image. Ho
 - there are no environment variables like `GOOGLE_PROJECT`, `WORKSPACE_NAME`, `WORKSPACE_BUCKET`, etc when running locally
 - there is no workspace-syncing when run locally
 
-To launch an image through Terra, navigate to https://app.terra.bio, select a workspace, enter your image in the "Custom Image" field, and click Create.
+To launch an image through Terra, navigate to https://app.terra.bio or your BEE's UI, select a workspace, enter your new image in the "Custom Image" field, and click Create.
 
 ## Automation Tests
 [Here](https://github.com/DataBiosphere/leonardo/tree/develop/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/notebooks) are automation tests for various docker image, please update the image hash for relevant tests. You can run the job build-terra-docker to automatically create a PR with your branch if you manually specify versions.

diff --git a/build_smoke_test_image.sh b/build_smoke_test_image.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # Perform a local build of an image for smoke-testing purposes. Also build its tree of base images, when missing.
 #
-# Example: ./build_smoke_test_image terra-jupyter-python  
+# Example: ./build_smoke_test_image.sh terra-jupyter-python  
 #
 # Current working directory must be the directory in which this script resides (one level above all the Dockerfiles).
 # It walks down each relevant image directory, making changes to the FROM statements,

diff --git a/config/community_images.json b/config/community_images.json
@@ -27,14 +27,4 @@
     "image": "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.16.1",
     "requiresSpark": false,
     "isRStudio": true
-},
-{
-    "id": "OpenVINO integration with Tensorflow",
-    "label": "OpenVINO integration with Tensorflow (openvino-tensorflow 1.1.0, Python 3.7.12, GATK 4.2.4.1)",
-    "version": "0.2.0",
-    "updated": "2022-01-31",
-    "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-gatk-ovtf-0.1.7-versions.json",
-    "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk-ovtf:0.1.7",
-    "requiresSpark": false,
-    "isCommunity": true
 }]
diff --git a/config/conf.json b/config/conf.json
@@ -26,7 +26,7 @@
                     "tidyverse"
                 ]
             },
-            "version" : "2.1.12",
+            "version" : "2.2.1",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : true,
@@ -47,7 +47,7 @@
                     "hail"
                 ]
             },
-            "version" : "1.0.27",
+            "version" : "1.1.1",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : true,
@@ -68,7 +68,7 @@
                     "scikit-learn"
                 ]
             },
-            "version" : "1.0.16",
+            "version" : "1.1.1",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : true,
@@ -86,7 +86,7 @@
             "packages" : {
 
             },
-            "version" : "1.0.15",
+            "version" : "1.1.1",
             "automated_flags" : {
                 "generate_docs" : true,
                 "include_in_ui" : false,
@@ -104,7 +104,7 @@
             "packages" : {
 
             },
-            "version" : "2.1.11",
+            "version" : "2.2.1",
             "automated_flags" : {
                 "include_in_ui" : false,
                 "generate_docs" : true,
@@ -124,7 +124,7 @@
             "packages" : {
 
             },
-            "version" : "2.2.14",
+            "version" : "2.3.1",
             "automated_flags" : {
                 "include_in_ui" : true,
                 "generate_docs" : true,
@@ -143,7 +143,7 @@
             "packages" : {
 
             },
-            "version" : "2.1.23",
+            "version" : "2.2.1",
             "automated_flags" : {
                 "include_in_ui" : false,
                 "generate_docs" : false,
@@ -171,21 +171,18 @@
             }
         },
         {
-            "name" : "terra-jupyter-gatk-ovtf",
-            "base_label" : "OpenVINO",
+            "name" : "wondershaper",
+            "base_label" : "wondershaper",
             "tools" : [
-                "gatk",
-                "python",
-                "r"
             ],
             "packages" : {
-                
+
             },
-            "version" : "0.2.10",
+            "version" : "0.0.1",
             "automated_flags" : {
                 "include_in_ui" : false,
-                "generate_docs" : true,
-                "build" : true,
+                "generate_docs" : false,
+                "build" : false,
                 "include_in_custom_dataproc" : false,
                 "include_in_custom_gce" : false
             }

diff --git a/config/legacy_static_images.json b/config/legacy_static_images.json
@@ -1,11 +1,20 @@
 [
   {
-    "updated": "2023-02-09",
-    "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.1.9",
+    "updated": "2023-06-23",
+    "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.1.11",
     "label": "Legacy R / Bioconductor (R 4.2.2, Bioconductor 3.16, Python 3.7.12)",
-    "version": "2.1.9",
+    "version": "2.1.11",
     "requiresSpark": false,
-    "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-bioconductor-2.1.9-versions.json",
+    "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-bioconductor-2.1.11-versions.json",
     "id": "terra-jupyter-bioconductor_legacy"
+  },
+  {
+    "updated": "2023-06-23",
+    "image": "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.2.14",
+    "label": "Legacy GATK (GATK 4.2.4.0, Python 3.7.12, R 4.3.0)",
+    "version": "2.2.14",
+    "requiresSpark": false,
+    "packages": "https://storage.googleapis.com/terra-docker-image-documentation/terra-jupyter-gatk:2.2.14-versions.json",
+    "id": "terra-jupyter-gatk_legacy"
   }
 ]