Move jsonnetfmt, jsonnet, goimports and others install into `to…

…ols/` (#2869)

* Move `jsonnetfmt` and `goimports` install into tools/

This will ensure that the version we are using to compile locally is the
same version that we are using in CI.  The `goimports` was already
included in the tools, and so the CI was redundant.  Also not, this
moves us from the C jsonnet to the Go jsonnet, which looks to introduce
slight formatting changes.

* Compile jsonnet

* Move tk, jb, and jsonnet also into tools/

* Include tools dependency on additional make targets

* Include tools Dockerfile

* Include tools image build target

* Update tools/go.sum

* Tidy up image build

* Prepare multi-arch docker image with install script

The script is used because not all imports in the tools.go are modules
which can be imported.  The version of the import is determined by
matching the nearest module path and installing the import at the
version specified in the go.mod.  This will allow updates to the go.mod
to direct the versions of the tools installed in the docker image.

* Stop ignoring the tools to allow docker build

* Include entrypoint

* Use TOOLS_CMD for docker execution on a few key make targets

* Use my image

* Fix variable

* Mark workdir as git safe

* Update drone to publish a tempo-ci-tools image

* Fix --platform for image source and use golang:alpine

.dockerignore

@@ -3,4 +3,3 @@ example/
 integration/
 operations/
 opentelemetry-proto/
-tools/

.drone/drone.jsonnet

@@ -114,11 +114,11 @@ local build_binaries(arch) = {
   ],
 };
 
-local docker_build(arch, app) = {
+local docker_build(arch, app, dockerfile='') = {
   name: 'build-%s-image' % app,
   image: 'plugins/docker',
   settings: {
-    dockerfile: 'cmd/%s/Dockerfile' % app,
+    dockerfile: if dockerfile != '' then dockerfile else 'cmd/%s/Dockerfile' % app,
     repo: 'grafana/%s' % app,
     username: { from_secret: docker_username_secret.name },
     password: { from_secret: docker_password_secret.name },
@@ -199,6 +199,29 @@ local deploy_to_dev() = {
       for arch in archs
     ],
   },
+] + [
+  // Publish tools Docker manifests
+  (
+    pipeline('docker-ci-tools-%s' % arch, arch) {
+      steps+: [
+        image_tag(arch),
+        docker_build(arch, 'tempo-ci-tools', dockerfile='tools/Dockerfile'),
+      ],
+    }
+  )
+  for arch in archs
+] + [
+  // Publish Docker manifests
+  pipeline('manifest-ci-tools') {
+    steps+: [
+      image_tag(),
+      docker_manifest('tempo-ci-tools'),
+    ],
+    depends_on+: [
+      'docker-ci-tools-%s' % arch
+      for arch in archs
+    ],
+  },
 ] + [
   // Continuously Deploy to dev env
   pipeline('cd-to-dev-env') {

.drone/drone.yml

@@ -171,6 +171,102 @@ trigger:
   - refs/heads/r??
   - refs/heads/r???
 ---
+depends_on: []
+kind: pipeline
+name: docker-ci-tools-amd64
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk --update --no-cache add bash
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-amd64 > .tags
+  image: alpine/git:v2.30.2
+  name: image-tag
+- image: plugins/docker
+  name: build-tempo-ci-tools-image
+  settings:
+    build_args:
+    - TARGETARCH=amd64
+    dockerfile: tools/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: grafana/tempo-ci-tools
+    username:
+      from_secret: docker_username
+trigger:
+  ref:
+  - refs/heads/main
+  - refs/tags/v*
+  - refs/heads/r?
+  - refs/heads/r??
+  - refs/heads/r???
+---
+depends_on: []
+kind: pipeline
+name: docker-ci-tools-arm64
+platform:
+  arch: arm64
+  os: linux
+steps:
+- commands:
+  - apk --update --no-cache add bash
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-arm64 > .tags
+  image: alpine/git:v2.30.2
+  name: image-tag
+- image: plugins/docker
+  name: build-tempo-ci-tools-image
+  settings:
+    build_args:
+    - TARGETARCH=arm64
+    dockerfile: tools/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: grafana/tempo-ci-tools
+    username:
+      from_secret: docker_username
+trigger:
+  ref:
+  - refs/heads/main
+  - refs/tags/v*
+  - refs/heads/r?
+  - refs/heads/r??
+  - refs/heads/r???
+---
+depends_on:
+- docker-ci-tools-amd64
+- docker-ci-tools-arm64
+kind: pipeline
+name: manifest-ci-tools
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk --update --no-cache add bash
+  - git fetch origin --tags
+  - echo $(./tools/image-tag) > .tags
+  image: alpine/git:v2.30.2
+  name: image-tag
+- image: plugins/manifest:1.4.0
+  name: manifest-tempo-ci-tools
+  settings:
+    password:
+      from_secret: docker_password
+    spec: .drone/docker-manifest.tmpl
+    target: tempo-ci-tools
+    username:
+      from_secret: docker_username
+trigger:
+  ref:
+  - refs/heads/main
+  - refs/tags/v*
+  - refs/heads/r?
+  - refs/heads/r??
+  - refs/heads/r???
+---
 depends_on:
 - manifest
 image_pull_secrets:
@@ -426,6 +522,6 @@ kind: secret
 name: gpg_passphrase
 ---
 kind: signature
-hmac: cdf6fce5bd5110303ec64ef2bb1d6aae70ed4b4df0f8de0a9eb7e3566a277c15
+hmac: 01adf0a8a3952d45aca5575e624a63a088167173e18c07b3dfe1950dcb3e069b
 
 ...

.github/workflows/ci.yml

@@ -27,13 +27,6 @@ jobs:
         with:
           go-version: 1.21.x
 
-      - name: Install jsonnetfmt and goimports
-        run: |
-          curl -fSL -o jsonnet.tar.gz https://github.com/google/jsonnet/releases/download/v0.17.0/jsonnet-bin-v0.17.0-linux.tar.gz
-          tar -xvf jsonnet.tar.gz -C /usr/local/bin/
-          chmod a+x /usr/local/bin/jsonnetfmt
-          go install golang.org/x/tools/cmd/[email protected]
-
       - name: Check out code
         uses: actions/checkout@v4
 
@@ -210,17 +203,6 @@ jobs:
         with:
           go-version: 1.21.x
 
-      - name: Install jsonnet, jsonnet-bundler & tanka
-        run: |
-          curl -fSL -o jsonnet.tar.gz https://github.com/google/jsonnet/releases/download/v0.17.0/jsonnet-bin-v0.17.0-linux.tar.gz
-          tar -xvf jsonnet.tar.gz -C /usr/local/bin/
-          chmod a+x /usr/local/bin/jsonnet
-
-          go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
-
-          curl -fSL -o /usr/local/bin/tk https://github.com/grafana/tanka/releases/download/v0.19.0/tk-linux-amd64
-          chmod a+x /usr/local/bin/tk
-
       - name: Check out code
         uses: actions/checkout@v4
 

Makefile

@@ -142,16 +142,16 @@ test-bench: tools docker-tempo
 	$(GOTEST) -v $(GOTEST_OPT) ./integration/bench
 
 .PHONY: fmt check-fmt
-fmt: tools
-	@gofumpt -w $(FILES_TO_FMT)
-	@goimports -w $(FILES_TO_FMT)
+fmt: tools-image
+	@$(TOOLS_CMD) gofumpt -w $(FILES_TO_FMT)
+	@$(TOOLS_CMD) goimports -w $(FILES_TO_FMT)
 
 check-fmt: fmt
 	@git diff --exit-code -- $(FILES_TO_FMT)
 
 .PHONY: jsonnetfmt check-jsonnetfmt
-jsonnetfmt:
-	@jsonnetfmt -i $(FILES_TO_JSONNETFMT)
+jsonnetfmt: tools-image
+	@$(TOOLS_CMD) jsonnetfmt -i $(FILES_TO_JSONNETFMT)
 
 check-jsonnetfmt: jsonnetfmt
 	@git diff --exit-code -- $(FILES_TO_JSONNETFMT)
@@ -302,14 +302,14 @@ docs-test:
 
 ### jsonnet
 .PHONY: jsonnet jsonnet-check jsonnet-test
-jsonnet:
-	$(MAKE) -C operations/jsonnet-compiled/util gen
+jsonnet: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util gen
 
-jsonnet-check:
-	$(MAKE) -C operations/jsonnet-compiled/util check
+jsonnet-check: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util check
 
-jsonnet-test:
-	$(MAKE) -C operations/jsonnet/microservices test
+jsonnet-test: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet/microservices test
 
 ### serverless
 .PHONY: docker-serverless test-serverless
@@ -321,11 +321,11 @@ test-serverless:
 
 ### tempo-mixin
 .PHONY: tempo-mixin tempo-mixin-check
-tempo-mixin:
-	$(MAKE) -C operations/tempo-mixin all
+tempo-mixin: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/tempo-mixin all
 
-tempo-mixin-check:
-	$(MAKE) -C operations/tempo-mixin check
+tempo-mixin-check: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/tempo-mixin check
 
 ### drone
 .PHONY: drone drone-jsonnet drone-signature

build/tools.mk

@@ -11,8 +11,26 @@ GO_MOD_OUTDATED  ?= go-mod-outdated
 TOOL_DIR     ?= tools
 TOOL_CONFIG  ?= $(TOOL_DIR)/tools.go
 
+TOOLS_IMAGE ?= zalegrala/tempo-ci-tools
+
 GOTOOLS ?= $(shell cd $(TOOL_DIR) && go list -e -f '{{ .Imports }}' -tags tools |tr -d '[]')
 
+TOOLS_CMD = docker run -v ${PWD}:/tools $(TOOLS_IMAGE)
+
+.PHONY: tools-image-build
+tools-image-build:
+	@echo "=== [ tools-image-build]: Building tools image..."
+	@docker build -t $(TOOLS_IMAGE) -f ./tools/Dockerfile .
+
+.PHONY: tools-docker
+tools-docker:
+	@echo "=== [ tools-docker     ]: Running tools in docker..."
+	@docker run -it -v $(shell pwd):/var/tempo $(TOOLS_IMAGE_NAME) make -C /var/tempo tools
+
+tools-image:
+	@echo "=== [ tools-image     ]: Running tools in docker..."
+	@docker pull ${TOOLS_IMAGE}
+
 tools:
 	@echo "=== [ tools            ]: Installing tools required by the project..."
 	@cd $(TOOL_DIR) && $(GO) install $(GOTOOLS)

operations/tempo-mixin-compiled/dashboards/tempo-operational.json

@@ -7112,4 +7112,4 @@
  "uid": "a6175b9cc7ec20591890117c39580030",
  "version": 1,
  "weekStart": ""
-}
+}

operations/tempo-mixin-compiled/dashboards/tempo-reads.json

@@ -1609,4 +1609,4 @@
  "title": "Tempo / Reads",
  "uid": "",
  "version": 0
-}
+}

operations/tempo-mixin-compiled/dashboards/tempo-resources.json

@@ -2428,4 +2428,4 @@
  "title": "Tempo / Resources",
  "uid": "",
  "version": 0
-}
+}

operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json

@@ -304,7 +304,7 @@
        },
        {
         "color": "orange",
-        "value": 0.20000000000000001
+        "value": 0.2
        },
        {
         "color": "red",
@@ -548,7 +548,7 @@
        },
        {
         "color": "orange",
-        "value": 0.20000000000000001
+        "value": 0.2
        },
        {
         "color": "red",
@@ -792,7 +792,7 @@
        },
        {
         "color": "red",
-        "value": 0.050000000000000003
+        "value": 0.05
        }
       ]
      },
@@ -1556,4 +1556,4 @@
  "title": "Rollout progress",
  "uid": "217b16a8c5966b32c770225dea289b19",
  "version": 0
-}
+}

operations/tempo-mixin-compiled/dashboards/tempo-tenants.json

@@ -1178,4 +1178,4 @@
  "title": "Tempo / Tenants",
  "uid": "",
  "version": 0
-}
+}

operations/tempo-mixin-compiled/dashboards/tempo-writes.json

@@ -1735,4 +1735,4 @@
  "title": "Tempo / Writes",
  "uid": "",
  "version": 0
-}
+}

tools/Dockerfile

@@ -0,0 +1,12 @@
+FROM --platform=$TARGETPLATFORM golang:alpine
+
+ARG TARGETOS
+ARG TARGETARCH
+ENV GOOS=$TARGETOS GOARCH=$TARGETARCH
+RUN apk --update add --no-cache make git bash
+
+WORKDIR /tools
+COPY tools /tools
+RUN /tools/install.sh
+COPY tools/entrypoint.sh /bin/entrypoint.sh
+ENTRYPOINT [ "/bin/entrypoint.sh" ]

tools/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh -eu
+
+if [ -n "${GITHUB_EMAIL+x}" ]; then
+	git config --global user.email "${GITHUB_EMAIL}"
+fi
+
+if [ -n "${GITHUB_NAME+x}" ]; then
+	git config --global user.name "${GITHUB_NAME}"
+fi
+
+git config --global --add safe.directory /tools
+
+exec "$@"