Merge branch 'upstream' into merge-upstream

dsh2dsh · Oct 21, 2024 · 7dbef18 · 7dbef18
2 parents 114b1f4 + 20abaa2
commit 7dbef18
Show file tree

Hide file tree

Showing 14 changed files with 108 additions and 211 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,16 +15,6 @@ commands:
               echo "$line" >> $BASH_ENV
             fi
 
-  invoke-lazy-sh:
-    parameters:
-      subcommand:
-        type: string
-    steps:
-      - run:
-          environment:
-            TERM: xterm
-          command: ./lazy.sh <<parameters.subcommand>>
-
   apt-update-and-install-common-deps:
     steps:
       - run: sudo apt-get update
@@ -34,18 +24,12 @@ commands:
       # The need for this was required for cimg/go:1.12, but let's future proof this here and now.
       - run: sudo apt-get install -y git ca-certificates
 
-  install-godep:
-    steps:
-      - apt-update-and-install-common-deps
-      - invoke-lazy-sh:
-          subcommand: godep
 
   install-docdep:
     steps:
       - apt-update-and-install-common-deps
       - run: sudo apt install python3 python3-pip libgirepository1.0-dev
-      - invoke-lazy-sh:
-          subcommand: docdep
+      - run: pip3 install -r docs/requirements.txt
 
   docs-publish-sh:
     parameters:
@@ -195,12 +179,25 @@ jobs:
 
       - go/load-cache:
           key: quickcheck-<<parameters.goversion>>
-      - install-godep
+      - run: make build/install
       - run: go mod download
       - run: cd build && go mod download
       - go/save-cache:
           key: quickcheck-<<parameters.goversion>>
 
+      # ensure all code has been generated
+      - run: make generate
+      - run: |
+          if output=$(git status --porcelain) && [ -z "$output" ]; then
+            echo "Working directory clean"
+          else
+            echo "Uncommitted changes"
+            echo ""
+            echo "$output"
+            exit 1
+          fi
+
+      # other checks
       - run: make formatcheck
       - run: make zrepl-bin test-platform-bin
       - run: make vet

diff --git a/Makefile b/Makefile
@@ -28,7 +28,6 @@ GO_MOD_READONLY := -mod=readonly
 GO_EXTRA_BUILDFLAGS :=
 GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
-GOLANGCI_LINT := golangci-lint
 GOCOVMERGE := gocovmerge
 RELEASE_GOVERSION ?= go1.23.1
 STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION))
@@ -71,7 +70,7 @@ release: ensure-release-toolchain
 	$(MAKE) noarch
 
 release-docker: $(ARTIFACTDIR) release-docker-mkcachemount
-	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
+	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build/build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
 	docker build -t zrepl_release --pull -f $(ARTIFACTDIR)/build.Dockerfile .
 	docker run --rm -i \
 		$(_RELEASE_DOCKER_CACHEMOUNT) \
@@ -244,8 +243,8 @@ _run_make_foreach_target_tuple:
 ##################### REGULAR TARGETS #####################
 .PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin
 
-lint:
-	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
+lint: build/install
+	$(GO_ENV_VARS) build/install/gobin/golangci-lint run ./...
 
 vet:
 	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
@@ -323,10 +322,27 @@ cover-full:
 # not part of the build, must do that manually
 .PHONY: generate formatcheck format
 
-generate:
-	protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
-	protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
-	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+build/install:
+	rm -rf build/install.tmp
+	mkdir build/install.tmp
+
+	-echo "installing protoc"
+	mkdir build/install.tmp/protoc
+	bash -x build/get_protoc.bash build/install.tmp/protoc
+
+	-echo "installing go tools"
+	build/go_install_tools.bash build/install.tmp/gobin
+
+	mv build/install.tmp build/install
+
+
+generate: build/install
+	# TODO: would be nice to run with a pure path here
+	PATH="$(CURDIR)/build/install/gobin:$(CURDIR)/build/install/protoc/bin:$$PATH" && \
+		build/install/protoc/bin/protoc -I=internal/replication/logic/pdu --go_out=internal/replication/logic/pdu --go-grpc_out=internal/replication/logic/pdu internal/replication/logic/pdu/pdu.proto && \
+		build/install/protoc/bin/protoc -I=internal/rpc/grpcclientidentity/example --go_out=internal/rpc/grpcclientidentity/example/pdu --go-grpc_out=internal/rpc/grpcclientidentity/example/pdu internal/rpc/grpcclientidentity/example/grpcauth.proto && \
+		$(GO) generate $(GO_BUILDFLAGS) -x ./... && \
+		true
 
 GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
 FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'

diff --git a/build.Dockerfile b/build.Dockerfile
diff --git a/build/.gitignore b/build/.gitignore
@@ -0,0 +1 @@
+install/
diff --git a/build/build.Dockerfile b/build/build.Dockerfile
@@ -0,0 +1,22 @@
+FROM !SUBSTITUTED_BY_MAKEFILE
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    unzip \
+    gawk
+
+# setup venv for docs
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ADD docs/requirements.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt
+
+# Go toolchain uses xdg-cache
+RUN mkdir -p /.cache && chmod -R 0777 /.cache
+
+# Go devtools are managed by Makefile
+
+WORKDIR /src
+
diff --git a/build.installprotoc.bash → build/get_protoc.bash b/build.installprotoc.bash → build/get_protoc.bash
@@ -2,6 +2,8 @@
 set -euo pipefail
 set -x
 
+cd "$1"
+
 MACH=$(uname -m)
 MACH="${MACH/aarch64/aarch_64}"
 
@@ -13,7 +15,7 @@ if [ -e "$FILENAME" ]; then
     exit 1
 fi
 
-wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
+wget --continue https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
 
 stat "$FILENAME"
 
@@ -22,4 +24,4 @@ d622619dcbfb5ecb281cfb92c1a74d6a0f42e752d9a2774b197f475f7ab1c8c4  protoc-28.0-li
 b2e187c8b9f2d97cd3ecae4926d1bb2cbebe3ab768e7c987cbc86bb17f319358  protoc-28.0-linux-x86_64.zip
 EOF
 
-unzip -d /usr/local "$FILENAME"
+unzip -d . "$FILENAME"
diff --git a/build/go_install_host_tool.source b/build/go_install_host_tool.source
@@ -0,0 +1,6 @@
+export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
+source <(go env)
+# build tools for the host platform
+export GOOS="$GOHOSTOS"
+export GOARCH="$GOHOSTARCH"
+# TODO GOARM=$GOHOSTARM?
diff --git a/build/go_install_tools.bash b/build/go_install_tools.bash
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+set -x
+
+OUTDIR="$(readlink -f "$1")"
+if [ -e "$OUTDIR" ]; then
+    echo "$OUTDIR" already exists 1>&2
+    exit 1
+fi
+
+# go install command below will install tools to $GOBIN
+export GOBIN="$OUTDIR"
+
+cd "$(dirname "$0")"
+
+source ./go_install_host_tool.source
+
+cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
diff --git a/docs/installation.rst b/docs/installation.rst
@@ -11,10 +11,9 @@ Installation
 
 .. toctree::
 
-    installation/user-privileges
     installation/packages
     installation/apt-repos
     installation/rpm-repos
-    installation/compile-from-source
+    installation/user-privileges
     installation/freebsd-jail-with-iocage
     installation/what-next
diff --git a/docs/installation/compile-from-source.rst b/docs/installation/compile-from-source.rst
diff --git a/docs/installation/packages.rst b/docs/installation/packages.rst
@@ -16,10 +16,13 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - OS / Distro
       - Install Command
       - Link
+    * - any
+      - Statically linked binaries.
+      - `Official GitHub releases <binary releases_>`_
     * - FreeBSD
       - ``pkg install zrepl``
       - `<https://www.freshports.org/sysutils/zrepl/>`_
-      
+
         :ref:`installation-freebsd-jail-with-iocage`
     * - FreeNAS
       -
@@ -30,7 +33,7 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - Arch Linux
       - ``yay install zrepl``
       - Available on `AUR <https://aur.archlinux.org/packages/zrepl>`_
-    * - Fedora, CentOS, RHEL, OpenSUSE
+    * - Fedora / RHEL / OpenSUSE
       - ``dnf install zrepl``
       - :ref:`RPM repository config <installation-rpm-repos>`
     * - Debian + Ubuntu
@@ -42,6 +45,6 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - Void Linux
       - ``xbps-install zrepl``
       - Available since `a88a2a4 <https://github.com/void-linux/void-packages/commit/a88a2a4d7bf56072dadf61ab56b8424e39155890>`_
-    * - Others
-      -
-      - Use `binary releases`_ or build from source.
+    * - any
+      - Build from source
+      - :repomasterlink:`README.md`.
diff --git a/internal/platformtest/tests/gen/wrapper.bash b/internal/platformtest/tests/gen/wrapper.bash
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+source ../../../build/go_install_host_tool.source
+
+GOOS=$GOHOSTOS GOARCH=$GOHOSTARCH go run ./gen ./
diff --git a/internal/platformtest/tests/tests.go b/internal/platformtest/tests/tests.go
@@ -13,4 +13,4 @@ func (c Case) String() string {
 	return runtime.FuncForPC(reflect.ValueOf(c).Pointer()).Name()
 }
 
-//go:generate ../../artifacts/generate-platform-test-list github.com/dsh2dsh/zrepl/platformtest/tests
+//go:generate ./gen/wrapper.bash