From 0954a4be4c40a7ac5041c4153a88fd6bef2a57aa Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot Date: Mon, 16 Dec 2024 19:59:22 +0100 Subject: [PATCH 1/3] Migrate tempo components docker workflow from Drone to GHA --- .github/workflows/docker.yml | 348 +++++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..bf9b5247084 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,348 @@ +# +#name: docker-amd64 +#platform: +# arch: amd64 +# os: linux +#steps: +#- commands: +# - apk --update --no-cache add bash +# - git fetch origin --tags +# - echo $(./tools/image-tag)-amd64 > .tags +# image: alpine/git:v2.30.2 +# name: image-tag +#- commands: +# - apk --update --no-cache add make git bash +# - COMPONENT=tempo GOARCH=amd64 make exe +# - COMPONENT=tempo-vulture GOARCH=amd64 make exe +# - COMPONENT=tempo-query GOARCH=amd64 make exe +# - COMPONENT=tempo-cli GOARCH=amd64 make exe +# image: golang:1.23-alpine +# name: build-tempo-binaries +#- image: plugins/docker +# name: build-tempo-image +# settings: +# build_args: +# - TARGETARCH=amd64 +# dockerfile: cmd/tempo/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/amd64 +# repo: grafana/tempo +# username: +# from_secret: docker_username +#- image: plugins/docker +# name: build-tempo-vulture-image +# settings: +# build_args: +# - TARGETARCH=amd64 +# dockerfile: cmd/tempo-vulture/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/amd64 +# repo: grafana/tempo-vulture +# username: +# from_secret: docker_username +#- image: plugins/docker +# name: build-tempo-query-image +# settings: +# build_args: +# - TARGETARCH=amd64 +# dockerfile: cmd/tempo-query/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/amd64 +# repo: grafana/tempo-query +# username: +# from_secret: docker_username +#- image: plugins/docker +# name: build-tempo-cli-image +# settings: +# build_args: +# - TARGETARCH=amd64 +# dockerfile: cmd/tempo-cli/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/amd64 +# repo: grafana/tempo-cli +# username: +# from_secret: docker_username +#trigger: +# ref: +# - refs/heads/main +# - refs/tags/v* +# - refs/heads/r? +# - refs/heads/r?? +# - refs/heads/r??? +#--- +#depends_on: [] +#kind: pipeline +#name: docker-arm64 +#platform: +# arch: arm64 +# os: linux +#steps: +#- commands: +# - apk --update --no-cache add bash +# - git fetch origin --tags +# - echo $(./tools/image-tag)-arm64 > .tags +# image: alpine/git:v2.30.2 +# name: image-tag +#- commands: +# - apk --update --no-cache add make git bash +# - COMPONENT=tempo GOARCH=arm64 make exe +# - COMPONENT=tempo-vulture GOARCH=arm64 make exe +# - COMPONENT=tempo-query GOARCH=arm64 make exe +# - COMPONENT=tempo-cli GOARCH=arm64 make exe +# image: golang:1.23-alpine +# name: build-tempo-binaries +#- image: plugins/docker +# name: build-tempo-image +# settings: +# build_args: +# - TARGETARCH=arm64 +# dockerfile: cmd/tempo/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/arm64 +# repo: grafana/tempo +# username: +# from_secret: docker_username +#- image: plugins/docker +# name: build-tempo-vulture-image +# settings: +# build_args: +# - TARGETARCH=arm64 +# dockerfile: cmd/tempo-vulture/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/arm64 +# repo: grafana/tempo-vulture +# username: +# from_secret: docker_username +#- image: plugins/docker +# name: build-tempo-query-image +# settings: +# build_args: +# - TARGETARCH=arm64 +# dockerfile: cmd/tempo-query/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/arm64 +# repo: grafana/tempo-query +# username: +# from_secret: docker_username +#- image: plugins/docker +# name: build-tempo-cli-image +# settings: +# build_args: +# - TARGETARCH=arm64 +# dockerfile: cmd/tempo-cli/Dockerfile +# password: +# from_secret: docker_password +# platform: linux/arm64 +# repo: grafana/tempo-cli +# username: +# from_secret: docker_username +#trigger: +# ref: +# - refs/heads/main +# - refs/tags/v* +# - refs/heads/r? +# - refs/heads/r?? +# - refs/heads/r??? +#--- +#depends_on: +#- docker-amd64 +#- docker-arm64 +#kind: pipeline +#name: manifest +#platform: +# arch: amd64 +# os: linux +#steps: +#- commands: +# - apk --update --no-cache add bash +# - git fetch origin --tags +# - echo $(./tools/image-tag) > .tags +# image: alpine/git:v2.30.2 +# name: image-tag +#- image: plugins/manifest:1.4.0 +# name: manifest-tempo +# settings: +# password: +# from_secret: docker_password +# spec: .drone/docker-manifest.tmpl +# target: tempo +# username: +# from_secret: docker_username +#- image: plugins/manifest:1.4.0 +# name: manifest-tempo-vulture +# settings: +# password: +# from_secret: docker_password +# spec: .drone/docker-manifest.tmpl +# target: tempo-vulture +# username: +# from_secret: docker_username +#- image: plugins/manifest:1.4.0 +# name: manifest-tempo-query +# settings: +# password: +# from_secret: docker_password +# spec: .drone/docker-manifest.tmpl +# target: tempo-query +# username: +# from_secret: docker_username +#- image: plugins/manifest:1.4.0 +# name: manifest-tempo-cli +# settings: +# password: +# from_secret: docker_password +# spec: .drone/docker-manifest.tmpl +# target: tempo-cli +# username: +# from_secret: docker_username +#trigger: +# ref: +# - refs/heads/main +# - refs/tags/v* +# - refs/heads/r? +# - refs/heads/r?? +# - refs/heads/r??? +#--- +# +##depends_on: +#- manifest +#image_pull_secrets: +#- dockerconfigjson +#kind: pipeline +#name: cd-to-dev-env +#platform: +# arch: amd64 +# os: linux +#steps: +#- commands: +# - apk --update --no-cache add bash +# - git fetch origin --tags +# - echo "grafana/tempo:$(./tools/image-tag)" > .tags-for-cd-tempo +# - echo "grafana/tempo-query:$(./tools/image-tag)" > .tags-for-cd-tempo_query +# - echo "grafana/tempo-vulture:$(./tools/image-tag)" > .tags-for-cd-tempo_vulture +#image: alpine/git:v2.30.2 +#name: image-tag-for-cd +#- image: us.gcr.io/kubernetes-dev/drone/plugins/updater +# name: update-dev-images +# settings: +# config_json: |- +# { +# "destination_branch": "master", +# "pull_request_branch_prefix": "auto-merge/cd-tempo-dev", +# "pull_request_enabled": true, +# "pull_request_existing_strategy": "ignore", +# "repo_name": "deployment_tools", +# "update_jsonnet_attribute_configs": [ +# { +# "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet", +# "jsonnet_key": "tempo", +# "jsonnet_value_file": ".tags-for-cd-tempo" +# }, +# { +# "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet", +# "jsonnet_key": "tempo_query", +# "jsonnet_value_file": ".tags-for-cd-tempo_query" +# }, +# { +# "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet", +# "jsonnet_key": "tempo_vulture", +# "jsonnet_value_file": ".tags-for-cd-tempo_vulture" +# } +# ] +# } +# github_app_id: +# from_secret: tempo_app_id_secret +# github_app_installation_id: +# from_secret: tempo_app_installation_id_secret +# github_app_private_key: +# from_secret: tempo_app_private_key_secret +#trigger: +# ref: +# - refs/heads/main + + +name: docker-ci-tools +on: + push: + branches: + - 'main' + +# Needed to login to DockerHub +permissions: + contents: read + id-token: write + +jobs: + + get-image-tag: + runs-on: ubuntu-24.04 + outputs: + tag: ${{ steps.get-tag.outputs.tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - id: get-tag + run: | + echo "tag=$(./tools/image-tag)" >> "$GITHUB_OUTPUT" + + docker: + needs: get-image-tag + strategy: + matrix: + component: [ tempo, tempo-vulture, tempo-query, tempo-cli ] + runner_arch: [ { runner: ubuntu-24.04, arch: amd64 }, { runner: github-hosted-ubuntu-arm64, arch: arm64 } ] + runs-on: ${{ matrix.runner_arch.runner }} + env: + TAG: ${{ needs.get-image-tag.outputs.tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to DockerHub + uses: grafana/shared-workflows/actions/dockerhub-login@dockerhub-login-v1.0.0 + + - name: build-tempo-binaries + run: | + COMPONENT=${{ matrix.component }} GOARCH=${{ matrix.runner_arch.arch }} make exe + + - name: docker-build-and-push-tempo + run: | + TAG_ARCH="$TAG-${{ matrix.runner_arch.arch }}" + docker build -f cmd/${{ matrix.component }}/Dockerfile -t grafana/${{ matrix.component }}:TAG_ARCH . + #docker push grafana/${{ matrix.component }}:TAG_ARCH + + - name: docker-manifest-create-and-push + run: | + docker manifest create \ + grafana/${{ matrix.component }}:$TAG \ + --amend grafana/${{ matrix.component }}:$TAG-amd64 \ + --amend grafana/${{ matrix.component }}:$TAG-arm64 + #docker manifest push grafana/${{ matrix.component }}:$TAG + + # to test + #docker manifest create \ + # grafana/${{ matrix.component }}:latest \ + # --amend grafana/${{ matrix.component }}:$TAG-amd64 \ + # --amend grafana/${{ matrix.component }}:$TAG-arm64 + #docker manifest push grafana/${{ matrix.component }}:latest + +# cd-to-dev-env: +# runs-on: ubuntu-24.04 +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# +# - name: get-tag +# run: | +# git fetch origin --tags +# echo "grafana/tempo:$(./tools/image-tag)" > .tags-for-cd-tempo +# echo "grafana/tempo-query:$(./tools/image-tag)" > .tags-for-cd-tempo_query +# echo "grafana/tempo-vulture:$(./tools/image-tag)" > .tags-for-cd-tempo_vulture From 026f504aa2e4741e07db232664be5995a2f63c8d Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot Date: Mon, 16 Dec 2024 20:35:21 +0100 Subject: [PATCH 2/3] wip --- .github/workflows/ci.yml | 198 ++++++--------------------------------- 1 file changed, 29 insertions(+), 169 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd148204783..cc279203d32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,6 @@ name: CI on: pull_request: -# Requiring certain checks for PRs to be merge-able in Github, forces for those checks to be *always* run. -# Even if the changes do not require them (i.e. the paths indicated below). That's why `paths-ignore` is commented out. -# -# paths-ignore: -# - 'docs/**' -# - 'example/**' -# - 'tools/**' concurrency: # Cancel any running workflow for the same branch when new commits are pushed. @@ -16,179 +9,46 @@ concurrency: group: "${{ github.ref_name }}-${{ github.head_ref }}" cancel-in-progress: true +# Needed to login to DockerHub +permissions: + contents: read + id-token: write + jobs: - lint: - name: Lint + get-image-tag: runs-on: ubuntu-24.04 + outputs: + tag: ${{ steps.get-tag.outputs.tag }} steps: - - - name: Check out code + - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - - name: check-fmt - run: make check-fmt - - - name: check-jsonnetfmt - run: make check-jsonnetfmt - - name: Get year and week number - id: get-year-week-number - run: echo "date=$(date +"%Yweek%U")" >> $GITHUB_OUTPUT - - - name: cache golangci - uses: actions/cache@v4 - with: - path: .cache/golangci-lint - key: golangci-lint-${{ runner.os }}-${{ steps.get-year-week-number.outputs.date }}-${{ hashFiles('go.mod', '.golangci.yml') }} - - - name: lint + - id: get-tag run: | - make lint base=origin/${{github.base_ref}} - sudo chown -R $(id -u):$(id -g) .cache/golangci-lint # needed to archive cache - - - unit-tests: - name: Run Unit Tests - runs-on: ubuntu-24.04 - strategy: - matrix: - test-target: - [ - test-with-cover-pkg, - test-with-cover-tempodb, - test-with-cover-tempodb-wal, - test-with-cover-others, - ] - - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" + echo "tag=$(./tools/image-tag)" >> "$GITHUB_OUTPUT" - - name: Run Tests - run: make ${{ matrix.test-target }} - - integration-tests: - name: Run integration tests - runs-on: ubuntu-24.04 + docker: + needs: get-image-tag strategy: matrix: - test-target: - [ - test-e2e, - test-integration-poller, - test-e2e-deployments, - test-e2e-serverless, - ] - - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Fetch tags - run: git fetch --prune --unshallow --tags - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - name: Run Tests - run: make ${{ matrix.test-target }} - - build: - name: Build - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - - name: Build Tempo - run: make tempo - - - name: generate-manifest - run: make generate-manifest - - - name: Build tempo-query - run: make tempo-query - - - name: Build vulture - run: make tempo-vulture - - - name: Build tempo-cli - run: make tempo-cli - - benchmark: - name: Benchmark - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - - name: Bench - run: make test-bench - - vendor-check: - name: Vendor check - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - - name: Install Protoc - uses: arduino/setup-protoc@v3 - - - name: Check vendor - run: make vendor-check - - tempo-jsonnet: - name: Check jsonnet & tempo-mixin - runs-on: ubuntu-24.04 + component: [ tempo, tempo-vulture, tempo-query, tempo-cli ] + runner_arch: [ { runner: ubuntu-24.04, arch: amd64 }, { runner: github-hosted-ubuntu-arm64, arch: arm64 } ] + runs-on: ${{ matrix.runner_arch.runner }} + env: + TAG: ${{ needs.get-image-tag.outputs.tag }} steps: - - name: Check out code + - name: Checkout uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - - name: Check jsonnet - run: make jsonnet-check + - name: Login to DockerHub + uses: grafana/shared-workflows/actions/dockerhub-login@dockerhub-login-v1.0.0 - - name: Check tempo-mixin - run: make tempo-mixin-check - - - name: Test jsonnet - run: make jsonnet-test + - name: build-tempo-binaries + run: | + COMPONENT=${{ matrix.component }} GOARCH=${{ matrix.runner_arch.arch }} make exe - build-technical-documentation: - name: Build technical documentation - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Build Website - run: docker run -v ${PWD}/docs/sources:/hugo/content/docs/tempo/latest --rm grafana/docs-base:latest make prod + - name: docker-build-and-push-tempo + run: | + TAG_ARCH="$TAG-${{ matrix.runner_arch.arch }}" + docker build -f cmd/${{ matrix.component }}/Dockerfile -t grafana/${{ matrix.component }}:TAG_ARCH . + #docker push grafana/${{ matrix.component }}:TAG_ARCH From bfdcad07c8f43c1bd6b15f96bb345137aaf41526 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot Date: Tue, 17 Dec 2024 13:09:16 +0100 Subject: [PATCH 3/3] wip --- .github/workflows/docker.yml | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bf9b5247084..b7be1e7a6bd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -346,3 +346,55 @@ jobs: # echo "grafana/tempo:$(./tools/image-tag)" > .tags-for-cd-tempo # echo "grafana/tempo-query:$(./tools/image-tag)" > .tags-for-cd-tempo_query # echo "grafana/tempo-vulture:$(./tools/image-tag)" > .tags-for-cd-tempo_vulture +# +# - name: Authenticate to GAR +# uses: grafana/shared-workflows/actions/login-to-gar@main +# id: login-to-gar +# with: +# registry: us-docker.pkg.dev +# environment: prod +# +# - name: Get Vault secrets +# uses: grafana/shared-workflows/actions/get-vault-secrets@main +# with: +# common_secrets: | +# GITHUB_APP_ID=updater-app:app-id +# GITHUB_APP_INSTALLATION_ID=updater-app:app-installation-id +# GITHUB_APP_PRIVATE_KEY=updater-app:private-key + +# - name: Update jsonnet +# run: | +# set -e -o pipefail +# +# cat << EOF > config.json +# { +# "destination_branch": "master", +# "pull_request_branch_prefix": "auto-merge/cd-tempo-dev", +# "pull_request_enabled": true, +# "pull_request_existing_strategy": "ignore", +# "repo_name": "deployment_tools", +# "update_jsonnet_attribute_configs": [ +# { +# "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet", +# "jsonnet_key": "tempo", +# "jsonnet_value_file": ".tags-for-cd-tempo" +# }, +# { +# "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet", +# "jsonnet_key": "tempo_query", +# "jsonnet_value_file": ".tags-for-cd-tempo_query" +# }, +# { +# "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet", +# "jsonnet_key": "tempo_vulture", +# "jsonnet_value_file": ".tags-for-cd-tempo_vulture" +# } +# ] +# } +# EOF +# +# docker run --rm \ +# -e GITHUB_APP_ID="$GITHUB_APP_ID" \ +# -e GITHUB_APP_INSTALLATION_ID="$GITHUB_APP_INSTALLATION_ID" \ +# -e GITHUB_APP_PRIVATE_KEY="$GITHUB_APP_PRIVATE_KEY" \ +# -e CONFIG_JSON="$(cat config.json)" us-docker.pkg.dev/grafanalabs-global/docker-deployment-tools-prod/updater