From dbeb1ee5132600ee47f25e677ca89a79e331597a Mon Sep 17 00:00:00 2001 From: Connor Barker Date: Wed, 18 Sep 2024 15:28:31 -0400 Subject: [PATCH] try it out --- .../juniper-eng-build-push-image/action.yaml | 68 +++++++++++ .github/workflows/beta-java-publish.yml | 115 ++++++++++++++++++ terraform/infra/.terraform.lock.hcl | 19 +++ terraform/infra/github_artifact_registry.tf | 63 ++++++++++ 4 files changed, 265 insertions(+) create mode 100644 .github/actions/juniper-eng-build-push-image/action.yaml create mode 100644 .github/workflows/beta-java-publish.yml create mode 100644 terraform/infra/github_artifact_registry.tf diff --git a/.github/actions/juniper-eng-build-push-image/action.yaml b/.github/actions/juniper-eng-build-push-image/action.yaml new file mode 100644 index 0000000000..478828ba34 --- /dev/null +++ b/.github/actions/juniper-eng-build-push-image/action.yaml @@ -0,0 +1,68 @@ +name: 'build and push image' +description: 'build and push docker images to Artifact Registry' +inputs: + version-tag: + required: true + description: version tag to append to docker images published by this workflow + image-repo: + required: true + description: URL for container image repo to publish to + image-name: + required: true + description: name of the application image to publish + gradle-build-args: + required: true + description: args to pass to the gradlew build command + publish-service-account: + required: false + description: email for the GCP service account used to publish app images + default: 'github-actions@broad-juniper-eng-infra.iam.gserviceaccount.com' +outputs: + published-image: + description: The full url and tag of the published image + value: ${{ steps.image-name.outputs.name }} +runs: + using: composite + steps: + - name: Setup JDK + uses: actions/setup-java@v3 + with: + java-version: '21' + distribution: 'adopt' + cache: 'gradle' + - name: Construct Docker Image Name and Tag + id: image-name + shell: bash + run: | + IMAGE_NAME="${{ inputs.image-repo }}/${{inputs.image-name }}:${{ inputs.version-tag }}" + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + - name: Auth to Google + uses: google-github-actions/auth@v1 + with: + # this value will always be the same so specifying directly + workload_identity_provider: projects/77554683012/locations/global/workloadIdentityPools/github-actions/providers/github-actions-provider + service_account: ${{ inputs.publish-service-account }} + + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@v1 + + - name: Explicitly Auth Docker For GCR + shell: bash + run: gcloud auth configure-docker ${{ inputs.image-repo }} --quiet + + - name: Build Image With Jib + shell: bash + run: | + ./gradlew --build-cache ${{ inputs.gradle-build-args }} \ + --image=${{ steps.image-name.outputs.name }} \ + -Djib.console=plain + + - name: Run Trivy Vulnerability Scan + uses: broadinstitute/dsp-appsec-trivy-action@v1 + with: + image: ${{ steps.image-name.outputs.name }} + + - name: Push Image + shell: bash + run: docker push ${{ steps.image-name.outputs.name }} + diff --git a/.github/workflows/beta-java-publish.yml b/.github/workflows/beta-java-publish.yml new file mode 100644 index 0000000000..141feba204 --- /dev/null +++ b/.github/workflows/beta-java-publish.yml @@ -0,0 +1,115 @@ +name: Publish and deploy +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + branches: + - cb-github-action-upload-to-gcr + +env: + SERVICE_NAME: ${{ github.event.repository.name }} + GOOGLE_PROJECT: broad-juniper-eng-infra + GOOGLE_DOCKER_REPOSITORY: us-central1-docker.pkg.dev + IMAGE_REPOSITORY_NAME: juniper + +jobs: + get-version-tag: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - name: Checkout Current Code + uses: actions/checkout@v3 + with: + token: ${{ secrets.BROADBOT_TOKEN }} + - name: Parse Tag + id: tag + run: echo "tag=$(git describe --tags)" >> $GITHUB_OUTPUT + + publish-admin-image: + needs: get-version-tag + permissions: + contents: 'read' + id-token: 'write' + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.build-publish.outputs.published-image }} + steps: + - name: Checkout Current Code + uses: actions/checkout@v3 + with: + token: ${{ secrets.BROADBOT_TOKEN }} + - name: build and publish image + id: build-publish + uses: ./.github/actions/juniper-eng-build-push-image + with: + version-tag: ${{ needs.get-version-tag.outputs.tag }} + image-repo: 'us-central1-docker.pkg.dev' + image-name: "broad-juniper-eng-infra/juniper/${{ github.event.repository.name }}-admin" + gradle-build-args: ':api-admin:jibDockerBuild' + + - name: Notify slack on failure + uses: broadinstitute/action-slack@v3.8.0 + if: failure() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + channel: '#juniper-dev-notifications' + status: failure + author_name: Publish docker Image + fields: job + text: "Publish to juniper eng infra failed :sadpanda:, image ${{ steps.build-publish.outputs.published-image }} failed to publish" + username: 'Juniper Build Notifications' + + publish-participant-image: + needs: get-version-tag + permissions: + contents: 'read' + id-token: 'write' + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.build-publish.outputs.published-image }} + steps: + - name: Checkout Current Code + uses: actions/checkout@v3 + with: + token: ${{ secrets.BROADBOT_TOKEN }} + - name: build and publish image + id: build-publish + uses: ./.github/actions/juniper-eng-build-push-image + with: + version-tag: ${{ needs.get-version-tag.outputs.tag }} + image-repo: 'us-central1-docker.pkg.dev' + image-name: "broad-juniper-eng-infra/juniper/${{ github.event.repository.name }}-participant" + gradle-build-args: ':api-participant:jibDockerBuild' + + - name: Notify slack on failure + uses: broadinstitute/action-slack@v3.8.0 + if: failure() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + channel: '#juniper-dev-notifications' + status: failure + author_name: Publish docker Image + fields: job + text: "Publish to broad-juniper-eng-infra failed :sadpanda:, image ${{ steps.build-publish.outputs.published-image }} failed to publish" + username: 'Juniper Build Notifications' + + notify-upon-completion: + runs-on: ubuntu-latest + if: always() + needs: [get-version-tag] + steps: + - uses: broadinstitute/action-slack@v3.8.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + channel: '#juniper-dev-notifications' + # Result status on the set version in dev job which actually performs the deploy + author_name: Image published to juniper eng infra + fields: job + text: Deploy to dev of ${{ needs.get-version-tag.outputs.tag }} completed successfully + username: 'Juniper Build Notifications' + + diff --git a/terraform/infra/.terraform.lock.hcl b/terraform/infra/.terraform.lock.hcl index cf87c7bcea..5fb78ed475 100644 --- a/terraform/infra/.terraform.lock.hcl +++ b/terraform/infra/.terraform.lock.hcl @@ -19,3 +19,22 @@ provider "registry.terraform.io/hashicorp/google" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "6.3.0" + hashes = [ + "h1:n9gyWvYz/1O5ecRfB1cV9/OlYTlSILvMy9knO4lvdXY=", + "zh:14911d94a78ee2a5d459a1e246e3722e04c5774f7aa8aeb415b11da89cd4ada8", + "zh:1a937609af8a837bdd181dad37e285c271bcb73971cc355c8f103730d1640aa4", + "zh:2032fa8f207ce06b5acf1e190f1090129b9fb4bae1b312165e5235af6cd61455", + "zh:3803fef01fe0c01e53f5d3628d8c8129d6443305cef458dea29d488661bbc5c0", + "zh:4c4434af7e2eb84c3a99a758e405b7f7f84c311ae6ef911d061d2d01fd4c0e76", + "zh:655eba025e121524ed95031c68b85d72ae2128eb748d68350fa4dd6569ce6770", + "zh:6678f186c550c6bb7ace33af0dd17a8d2ee6c61bcef2ff1bd8f603e0f7a5ce11", + "zh:7fa7de3cc6bf48d80ce3b8a5f3b23563d1f693b6febf20dd7b4a50a098ab6ee1", + "zh:a33d0f8deda840bc3f0263559118f632ddbb0609335739ab7329b517f156bc63", + "zh:c79b8109b3d7e42c1d1b5ebce29319bf69f4360be6f03f4fb0578fdc7f628e3f", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f9002acf0ce39e1f3f2afc28112343c846a6249868ca567477cbbcc3a4a77517", + ] +} diff --git a/terraform/infra/github_artifact_registry.tf b/terraform/infra/github_artifact_registry.tf new file mode 100644 index 0000000000..c2b52f2187 --- /dev/null +++ b/terraform/infra/github_artifact_registry.tf @@ -0,0 +1,63 @@ +# this file allows github to create artifacts in the artifact registry + + +# terraform translation of https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 + +# 0. create service account +resource "google_service_account" "github_actions" { + account_id = "github-actions" + display_name = "github-actions" + project = var.project +} + +# 1. create workload identity pool +resource "google_iam_workload_identity_pool" "github_actions_pool" { + workload_identity_pool_id = "github-actions" + provider = google-beta + project = var.project + display_name = "github-wip" +} + +# 2. create workload identity provider +resource "google_iam_workload_identity_pool_provider" "github_actions_pool_provider" { + workload_identity_pool_id = google_iam_workload_identity_pool.github_actions_pool.workload_identity_pool_id + workload_identity_pool_provider_id = "github-actions-provider" + display_name = "github-actions-provider" + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.actor" = "assertion.actor" + "attribute.aud" = "assertion.aud" + "attribute.repository" = "assertion.repository" + "attribute.repository_owner" = "assertion.repository_owner" + } + + # NOTE: this is what restricts external access, this ids are from github + attribute_condition = "assertion.repository_owner_id == '393552' && assertion.repository_id == '566938309'" + oidc { + allowed_audiences = [] + issuer_uri = "https://token.actions.githubusercontent.com" + } + +} + +# 3. create iam policy binding to connect the service account to the workload identity pool +resource "google_service_account_iam_binding" "github_sa_iam" { + service_account_id = google_service_account.github_actions.name + role = "roles/iam.workloadIdentityUser" + members = [ + "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github_actions_pool.name}/attribute.repository/broadinstitute/juniper" + ] +} + + + +# 4. create iam policy binding for access to GCR +resource "google_artifact_registry_repository_iam_binding" "github_artifact_registry_iam" { + repository = google_artifact_registry_repository.juniper_repo.name + location = var.region + project = var.project + role = "roles/artifactregistry.reader" + members = [ + "serviceAccount:${google_service_account.github_actions.email}" + ] +}