From 3df454681a4d688d6ffec15472e2bc6728df7002 Mon Sep 17 00:00:00 2001 From: bhoff Date: Thu, 31 Oct 2024 08:27:11 -0700 Subject: [PATCH 01/12] IT-3921: Rebuild container when Trivy code scan fails --- .github/workflows/trivy.yml | 3 +++ .github/workflows/trivy_periodic_image_scan.yml | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index da967cb..91c4a83 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -87,4 +87,7 @@ jobs: with: sarif_file: ${{ env.sarif_file_name }} wait-for-processing: true + + outputs: + trivy_conclusion: steps.trivy.outputs.conclusion ... diff --git a/.github/workflows/trivy_periodic_image_scan.yml b/.github/workflows/trivy_periodic_image_scan.yml index 1f57eef..f4429bb 100644 --- a/.github/workflows/trivy_periodic_image_scan.yml +++ b/.github/workflows/trivy_periodic_image_scan.yml @@ -32,4 +32,18 @@ jobs: # While GitHub repo's can be mixed (upper and lower) case, # Docker images can only be lower case IMAGE_NAME: ${{ needs.to-lower-case.outputs.lowercase-repo-name }} + EXIT_CODE: 1 + + # If scan failed, rebuild the image + update-image: + needs: periodic-scan + runs-on: ubuntu-latest + if: ${{needs.periodic-scan.outputs.trivy_conclusion == 'failure' }} + # tag the repo to trigger a new build + steps: + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v6.2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} ... From 64978da2d6547d2ce86778bcee547b2372c9749f Mon Sep 17 00:00:00 2001 From: bhoff Date: Thu, 31 Oct 2024 19:08:47 -0700 Subject: [PATCH 02/12] Need \!cancelled() && to make next job step run --- .github/workflows/trivy_periodic_image_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy_periodic_image_scan.yml b/.github/workflows/trivy_periodic_image_scan.yml index f4429bb..3ab181d 100644 --- a/.github/workflows/trivy_periodic_image_scan.yml +++ b/.github/workflows/trivy_periodic_image_scan.yml @@ -38,7 +38,7 @@ jobs: update-image: needs: periodic-scan runs-on: ubuntu-latest - if: ${{needs.periodic-scan.outputs.trivy_conclusion == 'failure' }} + if: ${{!cancelled() && needs.periodic-scan.outputs.trivy_conclusion == 'failure' }} # tag the repo to trigger a new build steps: - name: Bump version and push tag From 37871037568b47b4fde6c92616ca9468b1163ee9 Mon Sep 17 00:00:00 2001 From: bhoff Date: Tue, 5 Nov 2024 07:50:34 -0800 Subject: [PATCH 03/12] Run hourly on my fork as an experiment --- .github/workflows/trivy_periodic_image_scan.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trivy_periodic_image_scan.yml b/.github/workflows/trivy_periodic_image_scan.yml index 3ab181d..e3396c0 100644 --- a/.github/workflows/trivy_periodic_image_scan.yml +++ b/.github/workflows/trivy_periodic_image_scan.yml @@ -9,8 +9,10 @@ name: Trivy Periodic Image Scan on: schedule: + # run hourly (testing) + - cron: "0 * * * *" # run daily - - cron: "0 0 * * *" + #- cron: "0 0 * * *" jobs: to-lower-case: From 9b72766b2c84b9228a27739f5c1d5596bd192574 Mon Sep 17 00:00:00 2001 From: bhoff Date: Tue, 5 Nov 2024 08:26:06 -0800 Subject: [PATCH 04/12] Run daily on my fork --- .github/workflows/trivy_periodic_image_scan.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/trivy_periodic_image_scan.yml b/.github/workflows/trivy_periodic_image_scan.yml index e3396c0..3ab181d 100644 --- a/.github/workflows/trivy_periodic_image_scan.yml +++ b/.github/workflows/trivy_periodic_image_scan.yml @@ -9,10 +9,8 @@ name: Trivy Periodic Image Scan on: schedule: - # run hourly (testing) - - cron: "0 * * * *" # run daily - #- cron: "0 0 * * *" + - cron: "0 0 * * *" jobs: to-lower-case: From 3bd438f1f2a4fc59d99c3eeafbe7735aa79bdd61 Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 09:36:51 -0800 Subject: [PATCH 05/12] Build should fail if required packages fail to install --- Dockerfile | 5 +++-- install_packages_or_fail.R | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 install_packages_or_fail.R diff --git a/Dockerfile b/Dockerfile index 2233b02..fc3b852 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,10 @@ USER rstudio RUN python3 -m pip install virtualenv # Install R packages -RUN R -e "install.packages(c('tidyverse','devtools','BiocManager', 'reticulate'))" +ADD install_packages_or_die.R / +RUN Rscript --no-save install_packages_or_fail.R tidyverse devtools BiocManager reticulate # Install synapser and, by extension, the synapse Python client -RUN R -e "install.packages('synapser', repos=c('http://ran.synapse.org', 'http://cran.fhcrc.org'))" +RUN Rscript --no-save install_packages_or_fail.R synapser # Install Python package boto3, which will be used by the synapse Python client RUN R -e "reticulate::virtualenv_install(reticulate::virtualenv_list()[1], 'boto3')" diff --git a/install_packages_or_fail.R b/install_packages_or_fail.R new file mode 100644 index 0000000..a4ce7af --- /dev/null +++ b/install_packages_or_fail.R @@ -0,0 +1,14 @@ +#!/usr/bin/env Rscript +# from https://stackoverflow.com/questions/26244530/how-do-i-make-install-packages-return-an-error-if-an-r-package-cannot-be-install +# this script signals failure if a package fails to install + +packages = commandArgs(trailingOnly=TRUE) + +for (l in packages) { + + install.packages(l, dependencies=TRUE, repos=c('http://ran.synapse.org', 'https://cran.rstudio.com')) + + if ( ! library(l, character.only=TRUE, logical.return=TRUE) ) { + quit(status=1, save='no') + } +} \ No newline at end of file From 125d5f59175ab1cfa9b71738850a7534c3b1daeb Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 09:38:17 -0800 Subject: [PATCH 06/12] Build should fail if required packages fail to install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fc3b852..0a51a19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ USER rstudio RUN python3 -m pip install virtualenv # Install R packages -ADD install_packages_or_die.R / +ADD install_packages_or_fail.R / RUN Rscript --no-save install_packages_or_fail.R tidyverse devtools BiocManager reticulate # Install synapser and, by extension, the synapse Python client RUN Rscript --no-save install_packages_or_fail.R synapser From dfb0b98a50a3bd7757e2af9f1fdd32be3045f53a Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 09:39:48 -0800 Subject: [PATCH 07/12] white space --- install_packages_or_fail.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_packages_or_fail.R b/install_packages_or_fail.R index a4ce7af..3f57ac3 100644 --- a/install_packages_or_fail.R +++ b/install_packages_or_fail.R @@ -11,4 +11,4 @@ for (l in packages) { if ( ! library(l, character.only=TRUE, logical.return=TRUE) ) { quit(status=1, save='no') } -} \ No newline at end of file +} From 2f26571dc57fb8c733535572932afeb7168855f8 Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 13:43:04 -0800 Subject: [PATCH 08/12] IT-4004: Adding dependencies --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0a51a19..c7b7f22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,9 @@ ENV DISABLE_AUTH=true RUN apt-get -y update && \ apt-get -y upgrade && \ apt-get -y install libpng-dev \ +libcurl4-openssl-dev \ +libxml2-dev \ +libfontconfig1-dev \ python3 \ python3-pip \ python3-venv \ From e33f70ed45dbea604bf556af51de465b5e28d3a7 Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 14:10:56 -0800 Subject: [PATCH 09/12] IT-4004 adding more dependencies --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index c7b7f22..6ceae0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,13 @@ apt-get -y install libpng-dev \ libcurl4-openssl-dev \ libxml2-dev \ libfontconfig1-dev \ +libgit2-dev \ +libfontconfig1-dev \ +libfribidi-dev \ +libfreetype6-dev \ +libpng-dev \ +libtiff5-dev \ +libjpeg-dev \ python3 \ python3-pip \ python3-venv \ From 96dc3d461c5b9256d44f9280cd01aa48293fcf67 Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 14:33:21 -0800 Subject: [PATCH 10/12] IT-4004: Add dependency; install versioned rjson --- Dockerfile | 4 ++++ install_versioned_package_or_fail.R | 14 ++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 install_versioned_package_or_fail.R diff --git a/Dockerfile b/Dockerfile index 6ceae0b..08333e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ libfreetype6-dev \ libpng-dev \ libtiff5-dev \ libjpeg-dev \ +libharfbuzz-dev \ python3 \ python3-pip \ python3-venv \ @@ -31,6 +32,9 @@ RUN python3 -m pip install virtualenv # Install R packages ADD install_packages_or_fail.R / +ADD install_versioned_package_or_fail.R / +# synapser depends on rjson 0.2.21, but a newer version is installed by default +RUN Rscript --no-save install_versioned_package_or_fail.R rjson 0.2.21 RUN Rscript --no-save install_packages_or_fail.R tidyverse devtools BiocManager reticulate # Install synapser and, by extension, the synapse Python client RUN Rscript --no-save install_packages_or_fail.R synapser diff --git a/install_versioned_package_or_fail.R b/install_versioned_package_or_fail.R new file mode 100644 index 0000000..2c1989f --- /dev/null +++ b/install_versioned_package_or_fail.R @@ -0,0 +1,14 @@ +#!/usr/bin/env Rscript +# from https://stackoverflow.com/questions/26244530/how-do-i-make-install-packages-return-an-error-if-an-r-package-cannot-be-install +# this script signals failure if a package fails to install + +theargs = commandArgs(trailingOnly=TRUE) + +package=theargs[1] +version=theargs[2] + +remotes::install_version(package, version = version, repos = 'https://cran.rstudio.com') + +if ( ! library(package, character.only=TRUE, logical.return=TRUE) ) { + quit(status=1, save='no') +} From 95acabf80bb573c9830f8ff46ec74b5fe568a3fa Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 14:38:54 -0800 Subject: [PATCH 11/12] IT-4004: Add dependency; install versioned rjson --- install_versioned_package_or_fail.R | 1 + 1 file changed, 1 insertion(+) diff --git a/install_versioned_package_or_fail.R b/install_versioned_package_or_fail.R index 2c1989f..7cc1dd3 100644 --- a/install_versioned_package_or_fail.R +++ b/install_versioned_package_or_fail.R @@ -7,6 +7,7 @@ theargs = commandArgs(trailingOnly=TRUE) package=theargs[1] version=theargs[2] +install.packages('remotes', dependencies=TRUE, repos='https://cran.rstudio.com') remotes::install_version(package, version = version, repos = 'https://cran.rstudio.com') if ( ! library(package, character.only=TRUE, logical.return=TRUE) ) { From acd6286f6e240ad369784aa88cba667463352723 Mon Sep 17 00:00:00 2001 From: bhoff Date: Wed, 13 Nov 2024 15:19:33 -0800 Subject: [PATCH 12/12] IT-4004: Updated comments --- install_packages_or_fail.R | 3 ++- install_versioned_package_or_fail.R | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/install_packages_or_fail.R b/install_packages_or_fail.R index 3f57ac3..7ff139a 100644 --- a/install_packages_or_fail.R +++ b/install_packages_or_fail.R @@ -1,6 +1,7 @@ #!/usr/bin/env Rscript # from https://stackoverflow.com/questions/26244530/how-do-i-make-install-packages-return-an-error-if-an-r-package-cannot-be-install -# this script signals failure if a package fails to install +# install the latest versions of a list of packages and fail +# if any package fails to install packages = commandArgs(trailingOnly=TRUE) diff --git a/install_versioned_package_or_fail.R b/install_versioned_package_or_fail.R index 7cc1dd3..c1b824f 100644 --- a/install_versioned_package_or_fail.R +++ b/install_versioned_package_or_fail.R @@ -1,6 +1,6 @@ #!/usr/bin/env Rscript -# from https://stackoverflow.com/questions/26244530/how-do-i-make-install-packages-return-an-error-if-an-r-package-cannot-be-install -# this script signals failure if a package fails to install +# install a specific version of a given package and fail if +# the package fails to install theargs = commandArgs(trailingOnly=TRUE)