Skip to content

Commit

Permalink
Extract reusable publish workflow (#1007)
Browse files Browse the repository at this point in the history
- Combine release, prerelease and main into a single workflow with
minimal options.
- This enables verifying all releases except Go on main. We can't verify
Go because we don't release Go because that would clutter the git
history with tags.

## Prerelease -> Release differences

Here's a diff between `prerelease.yml` and `release.yml` before the
change. The observed changes are:
1. Extra `IS_PRERELEASE: true` env var
2. Optional extra `create_docs_build` job in `release.yml`.
3. Customised goreleaser args
4. `enableMacosRunner` is disabled for pre-release.
5. Workflow name and tags filters.

```diff
4d3
<   IS_PRERELEASE: true
22a22,40
>   #{{ if .Config.publishRegistry -}}#
>   create_docs_build:
>     name: create_docs_build
>     needs: publish_go_sdk
>     runs-on: #{{ .Config.runner.default }}#
>     steps:
>       - name: Dispatch Metadata build
>         uses: peter-evans/repository-dispatch@v3
>         with:
>           token: ${{ secrets.PULUMI_BOT_TOKEN }}
>           repository: pulumi/registry
>           event-type: resource-provider
>           client-payload: |-
>             {
>               "project": "${{ github.repository }}",
>               "project-shortname": "#{{ .Config.provider }}#",
>               "ref": "${{ github.ref_name }}"
>             }
>   #{{ end -}}#
79,80c97
<         args: -p #{{ .Config.parallel }}# -f .goreleaser.prerelease.yml --rm-dist --skip-validate --timeout
<           #{{ .Config.timeout }}#m0s
---
>         args: -p #{{ .Config.parallel }}# release --rm-dist --timeout #{{ .Config.timeout }}#m0s
138a156,176
> 
>   clean_up_release_labels:
>     name: Clean up release labels
>     #{{ if .Config.publishRegistry -}}#
>     needs: create_docs_build
>     #{{ else }}#
>     needs: tag_sdk
>     #{{- end }}#
>     runs-on: #{{ .Config.runner.default }}#
>     steps:
>     - name: Checkout Repo
>       uses: actions/checkout@v4
>     - name: Clean up release labels
>       uses: pulumi/action-release-by-pr-label@main
>       with:
>         command: "clean-up-release-labels"
>         repo: ${{ github.repository }}
>         commit: ${{ github.sha }}
>       env:
>         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
> 
255,256c293
<       # Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
<       enableMacosRunner: false
---
>       enableMacosRunner: true
258c295
< name: prerelease
---
> name: release
262c299,300
<     - v*.*.*-**
---
>     - v*.*.*
>     - "!v*.*.*-**"
```

## Main -> Prerelease differences

Diff summary:

1. `IS_PRERELEASE` was not set for main.
2. `generate_coverage_data` is only run on main - this is not part of
the publish workflow and remains unchanged.
3. `tag_release_if_labeled_needs_release` in only in main - this will
remain unchanged.
4. `publish_go_sdk` was not in main - we will skip this in the publish
workflow.
5. `verify-release` was not in main - we can add this but skip go on
main as we're not publishing it.
6. The triggers & workflow name vary.

```diff
3a4
>   IS_PRERELEASE: true
22,63d22
<   generate_coverage_data:
<     continue-on-error: true
<     env:
<       COVERAGE_OUTPUT_DIR: ${{ secrets.COVERAGE_OUTPUT_DIR }}
<     name: generate_coverage_data
<     needs: prerequisites
<     runs-on: #{{ .Config.runner.default }}#
<     steps:
<     - name: Free Disk Space (Ubuntu)
<       uses: jlumbroso/[email protected]
<       with:
<         tool-cache: false
<         swap-storage: false
<     - name: Checkout Repo
<       uses: #{{ .Config.actionVersions.checkout }}#
< #{{- if .Config.checkoutSubmodules }}#
<       with:
<         submodules: #{{ .Config.checkoutSubmodules }}#
< #{{- end }}#
<     - name: Configure AWS Credentials
<       uses: #{{ .Config.actionVersions.configureAwsCredentials }}#
<       with:
<         aws-access-key-id: ${{ secrets.AWS_CORP_S3_UPLOAD_ACCESS_KEY_ID }}
<         aws-region: us-west-2
<         aws-secret-access-key: ${{ secrets.AWS_CORP_S3_UPLOAD_SECRET_ACCESS_KEY }}
<     - name: Setup tools
<       uses: ./.github/actions/setup-tools
<       with:
<         tools: pulumictl, pulumicli, go, schema-tools
<     - name: Echo Coverage Output Dir
<       run: 'echo "Coverage output directory: ${{ env.COVERAGE_OUTPUT_DIR }}"'
<     - name: Generate Coverage Data
<       run: PULUMI_MISSING_DOCS_ERROR=true make tfgen
<     - name: Summarize Provider Coverage Results
<       run: cat ${{ env.COVERAGE_OUTPUT_DIR }}/shortSummary.txt
<     - name: Upload coverage data to S3
<       run: >-
<         summaryName="${PROVIDER}_summary_$(date +"%Y-%m-%d_%H-%M-%S").json"
< 
<         s3FullURI="s3://${{ secrets.S3_COVERAGE_BUCKET_NAME }}/summaries/${summaryName}"
< 
<         aws s3 cp "${{ env.COVERAGE_OUTPUT_DIR }}/summary.json" "${s3FullURI}" --acl bucket-owner-full-control
123d81
< 
146,149c104,108
< 
<   tag_release_if_labeled_needs_release:
<     name: Tag release if labeled as needs-release
<     needs: publish_sdk
---
>   publish_go_sdk:
>     name: publish_go_sdk
>     needs: 
>       - prerequisites
>       - publish_sdk
152,153c111,113
<     - name: check if this commit needs release
<       uses: pulumi/action-release-by-pr-label@main
---
>     - name: Checkout Repo
>       uses: #{{ .Config.actionVersions.checkout }}#
> #{{- if .Config.checkoutSubmodules }}#
155,163c115,138
<         command: "release-if-needed"
<         repo: ${{ github.repository }}
<         commit: ${{ github.sha }}
<         slack_channel: ${{ secrets.RELEASE_OPS_SLACK_CHANNEL }}
<       env:
<         RELEASE_BOT_ENDPOINT: ${{ secrets.RELEASE_BOT_ENDPOINT }}
<         RELEASE_BOT_KEY: ${{ secrets.RELEASE_BOT_KEY }}
<         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
< 
---
>         submodules: #{{ .Config.checkoutSubmodules }}#
> #{{- end }}#
>     - name: Download Go SDK
>       uses: actions/download-artifact@v4
>       with:
>         name: go-sdk.tar.gz
>         path: ${{ github.workspace }}/sdk/
>     - name: Uncompress Go SDK
>       run: tar -zxf ${{ github.workspace }}/sdk/go.tar.gz -C
>         ${{ github.workspace }}/sdk/go
>       shell: bash
>     - uses: pulumi/publish-go-sdk-action@v1
>       with:
>         repository: ${{ github.repository }}
>         base-ref: ${{ github.sha }}
>         source: sdk
>         path: sdk
>         version: ${{ needs.prerequisites.outputs.version }}
>         additive: false
>         # Avoid including other language SDKs & artifacts in the commit
>         files: |
>           go.*
>           go/**
>           !*.tar.gz
269c244,258
< name: #{{ .Config.providerDefaultBranch }}#
---
>   verify-release:
>     name: verify-release
>     needs:
>       - prerequisites
>       - publish
>       - publish_sdk
>       - publish_go_sdk
>     uses: ./.github/workflows/verify-release.yml
>     secrets: inherit
>     with:
>       providerVersion: ${{ needs.prerequisites.outputs.version }}
>       # Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
>       enableMacosRunner: false
> 
> name: prerelease
272,279c261,262
<     branches:
<     - #{{ .Config.providerDefaultBranch }}#
<     paths-ignore:
<     - "**.md"
<     tags-ignore:
<     - v*
<     - sdk/*
<     - "**"
---
>     tags:
>     - v*.*.*-**
```
  • Loading branch information
danielrbradley authored Jun 28, 2024
1 parent 59792d5 commit 5721995
Show file tree
Hide file tree
Showing 20 changed files with 812 additions and 1,165 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -82,71 +82,16 @@ jobs:
#{{- range $action, $_ := .Config.extraTests }}#
- #{{ $action }}#
#{{- end }}#
runs-on: #{{ if .Config.runner.publish }}##{{- .Config.runner.publish }}##{{ else }}##{{- .Config.runner.default }}##{{ end }}#
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/[email protected]
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
swap-storage: false
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Setup tools
uses: ./.github/actions/setup-tools
with:
tools: pulumictl, pulumicli, go
- name: Configure AWS Credentials
uses: #{{ .Config.actionVersions.configureAwsCredentials }}#
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-region: us-east-2
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
role-duration-seconds: 7200
role-external-id: upload-pulumi-release
role-session-name: #{{ .Config.provider }}#@githubActions
role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE_ARN }}
- name: Run GoReleaser
uses: #{{ .Config.actionVersions.goReleaser }}#
env:
GORELEASER_CURRENT_TAG: v${{ needs.prerequisites.outputs.version }}
PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }}
with:
args: -p #{{ .Config.parallel }}# -f .goreleaser.prerelease.yml --rm-dist --skip-validate --timeout
#{{ .Config.timeout }}#m0s
version: latest

publish_sdk:
name: publish_sdk
needs:
- prerequisites
- publish
runs-on: #{{ .Config.runner.default }}#
steps:
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Setup tools
uses: ./.github/actions/setup-tools
with:
tools: pulumicli, pulumictl, go, node, dotnet, python, java
- name: Publish SDKs
uses: #{{ .Config.publish.publisherAction }}#
with:
sdk: #{{ .Config.publish.sdk }}#
version: ${{ needs.prerequisites.outputs.version }}
uses: ./.github/workflows/publish.yml
secrets: inherit
with:
version: ${{ needs.prerequisites.outputs.version }}
isPrerelease: true
skipGoSdk: true

tag_release_if_labeled_needs_release:
name: Tag release if labeled as needs-release
needs: publish_sdk
needs: publish
runs-on: #{{ .Config.runner.default }}#
steps:
- name: check if this commit needs release
Expand Down Expand Up @@ -268,6 +213,7 @@ jobs:
#{{ end }}#
name: #{{ .Config.providerDefaultBranch }}#
on:
workflow_dispatch: {}
push:
branches:
- #{{ .Config.providerDefaultBranch }}#
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,101 +41,12 @@ jobs:
#{{- range $action, $_ := .Config.extraTests }}#
- #{{ $action }}#
#{{- end }}#
runs-on: #{{ if .Config.runner.publish }}##{{- .Config.runner.publish }}##{{ else }}##{{- .Config.runner.default }}##{{ end }}#
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/[email protected]
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
swap-storage: false
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Setup tools
uses: ./.github/actions/setup-tools
with:
tools: pulumictl, pulumicli, go
- name: Configure AWS Credentials
uses: #{{ .Config.actionVersions.configureAwsCredentials }}#
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-region: us-east-2
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
role-duration-seconds: 7200
role-external-id: upload-pulumi-release
role-session-name: #{{ .Config.provider }}#@githubActions
role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE_ARN }}
- name: Run GoReleaser
uses: #{{ .Config.actionVersions.goReleaser }}#
env:
GORELEASER_CURRENT_TAG: v${{ needs.prerequisites.outputs.version }}
PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }}
with:
args: -p #{{ .Config.parallel }}# -f .goreleaser.prerelease.yml --rm-dist --skip-validate --timeout
#{{ .Config.timeout }}#m0s
version: latest
publish_sdk:
name: publish_sdk
needs:
- prerequisites
- publish
runs-on: #{{ .Config.runner.default }}#
steps:
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Setup tools
uses: ./.github/actions/setup-tools
with:
tools: pulumicli, pulumictl, go, node, dotnet, python, java
- name: Publish SDKs
uses: #{{ .Config.publish.publisherAction }}#
with:
sdk: #{{ .Config.publish.sdk }}#
version: ${{ needs.prerequisites.outputs.version }}
publish_go_sdk:
name: publish_go_sdk
needs:
- prerequisites
- publish_sdk
runs-on: #{{ .Config.runner.default }}#
steps:
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Download Go SDK
uses: actions/download-artifact@v4
with:
name: go-sdk.tar.gz
path: ${{ github.workspace }}/sdk/
- name: Uncompress Go SDK
run: tar -zxf ${{ github.workspace }}/sdk/go.tar.gz -C
${{ github.workspace }}/sdk/go
shell: bash
- uses: pulumi/publish-go-sdk-action@v1
with:
repository: ${{ github.repository }}
base-ref: ${{ github.sha }}
source: sdk
path: sdk
version: ${{ needs.prerequisites.outputs.version }}
additive: false
# Avoid including other language SDKs & artifacts in the commit
files: |
go.*
go/**
!*.tar.gz
uses: ./.github/workflows/publish.yml
secrets: inherit
with:
version: ${{ needs.prerequisites.outputs.version }}
isPrerelease: true

test:
name: test
needs:
Expand Down Expand Up @@ -241,19 +152,6 @@ jobs:
#{{- if .Config.extraTests }}#
#{{ .Config.extraTests | toYaml | indent 2 }}#
#{{ end }}#
verify-release:
name: verify-release
needs:
- prerequisites
- publish
- publish_sdk
- publish_go_sdk
uses: ./.github/workflows/verify-release.yml
secrets: inherit
with:
providerVersion: ${{ needs.prerequisites.outputs.version }}
# Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
enableMacosRunner: false

name: prerelease
on:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
# WARNING: This file is autogenerated - changes will be overwritten if not made via https://github.com/pulumi/ci-mgmt
name: Publish

on:
workflow_call:
inputs:
version:
required: true
type: string
isPrerelease:
required: true
type: boolean
skipGoSdk:
default: false
type: boolean
description: Skip publishing & verifying the Go SDK

env:
IS_PRERELEASE: ${{ inputs.isPrerelease }}
#{{ .Config.env | toYaml | indent 2 }}#

jobs:
publish:
name: publish
runs-on: #{{ if .Config.runner.publish }}##{{- .Config.runner.publish }}##{{ else }}##{{- .Config.runner.default }}##{{ end }}#
steps:
- name: Validate prerelease
if: inputs.isPrerelease == false && (contains(inputs.version, '-') || contains(inputs.version, '+'))
run: echo "Can't publish a prerelease version as a stable release. This is likely a bug in the calling workflow." && exit 1
- name: Validate skipGoSdk
if: inputs.skipGoSdk && inputs.isPrerelease == false
run: echo "Can't skip Go SDK for stable releases. This is likely a bug in the calling workflow." && exit 1
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/[email protected]
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
swap-storage: false
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Setup tools
uses: ./.github/actions/setup-tools
with:
tools: pulumictl, pulumicli, go
- name: Configure AWS Credentials
uses: #{{ .Config.actionVersions.configureAwsCredentials }}#
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-region: us-east-2
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
role-duration-seconds: 7200
role-external-id: upload-pulumi-release
role-session-name: #{{ .Config.provider }}#@githubActions
role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE_ARN }}
- name: Run GoReleaser
if: inputs.isPrerelease == false
uses: #{{ .Config.actionVersions.goReleaser }}#
env:
GORELEASER_CURRENT_TAG: v${{ inputs.version }}
PROVIDER_VERSION: ${{ inputs.version }}
with:
args: -p #{{ .Config.parallel }}# release --rm-dist --timeout #{{ .Config.timeout }}#m0s
version: latest
- name: Run GoReleaser (prerelease)
if: inputs.isPrerelease == true
uses: #{{ .Config.actionVersions.goReleaser }}#
env:
GORELEASER_CURRENT_TAG: v${{ inputs.version }}
PROVIDER_VERSION: ${{ inputs.version }}
with:
args: -p #{{ .Config.parallel }}# -f .goreleaser.prerelease.yml --rm-dist --skip-validate --timeout
#{{ .Config.timeout }}#m0s
version: latest

publish_sdk:
name: publish_sdk
needs: publish
runs-on: #{{ .Config.runner.default }}#
steps:
- name: Checkout Repo
uses: #{{ .Config.actionVersions.checkout }}#
#{{- if .Config.checkoutSubmodules }}#
with:
submodules: #{{ .Config.checkoutSubmodules }}#
#{{- end }}#
- name: Setup tools
uses: ./.github/actions/setup-tools
with:
tools: pulumictl, pulumicli, go, node, dotnet, python, java
- name: Publish SDKs
uses: pulumi/[email protected]
with:
sdk: #{{ .Config.publish.sdk }}#
version: ${{ inputs.version }}
- name: Download Go SDK
uses: ./.github/actions/download-sdk
with:
language: go
- uses: pulumi/publish-go-sdk-action@v1
if: inputs.skipGoSdk == false
with:
repository: ${{ github.repository }}
base-ref: ${{ github.sha }}
source: sdk
path: sdk
version: ${{ inputs.version }}
additive: false
# Avoid including other language SDKs & artifacts in the commit
files: |
go.*
go/**
!*.tar.gz
#{{- if .Config.publishRegistry }}#
create_docs_build:
name: create_docs_build
needs: publish_sdk
# Only run for non-prerelease, if the publish_go_sdk job was successful or skipped
if: inputs.isPrerelease == false
runs-on: #{{ .Config.runner.default }}#
steps:
- name: Dispatch Metadata build
uses: peter-evans/repository-dispatch@v3
with:
token: ${{ secrets.PULUMI_BOT_TOKEN }}
repository: pulumi/registry
event-type: resource-provider
client-payload: |-
{
"project": "${{ github.repository }}",
"project-shortname": "#{{ .Config.provider }}#",
"ref": "${{ github.ref_name }}"
}
#{{- end }}#

clean_up_release_labels:
name: Clean up release labels
# Only run for non-prerelease, if the publish_go_sdk job was successful or skipped
if: inputs.isPrerelease == false
#{{ if .Config.publishRegistry -}}#
needs: create_docs_build
#{{ else }}#
needs: publish_sdk
#{{- end }}#
runs-on: #{{ .Config.runner.default }}#
steps:
- name: Checkout Repo
uses: actions/checkout@v4
- name: Clean up release labels
uses: pulumi/action-release-by-pr-label@main
with:
command: "clean-up-release-labels"
repo: ${{ github.repository }}
commit: ${{ github.sha }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

verify_release:
name: verify_release
needs: publish_sdk
uses: ./.github/workflows/verify-release.yml
secrets: inherit
with:
providerVersion: ${{ inputs.version }}
# Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
enableMacosRunner: ${{ inputs.isPrerelease == false }}
skipGoSdk: ${{ inputs.skipGoSdk }}
Loading

0 comments on commit 5721995

Please sign in to comment.