From 752d16e4fc1628f75b7bde7c5d3983ff7c4fac5f Mon Sep 17 00:00:00 2001 From: ChandraP <4702817+chandrareddyp@users.noreply.github.com> Date: Fri, 9 Feb 2024 08:18:38 -0600 Subject: [PATCH] rebase --- .../cli_core_e2e_windows_test_msys2.yaml | 69 ++++++++--- Makefile | 114 +++++++++++++++--- hack/tools/Makefile | 74 ++++++++---- 3 files changed, 204 insertions(+), 53 deletions(-) diff --git a/.github/workflows/cli_core_e2e_windows_test_msys2.yaml b/.github/workflows/cli_core_e2e_windows_test_msys2.yaml index 1790fb646..19b57b8f3 100644 --- a/.github/workflows/cli_core_e2e_windows_test_msys2.yaml +++ b/.github/workflows/cli_core_e2e_windows_test_msys2.yaml @@ -41,6 +41,35 @@ jobs: restore-keys: | ${{ runner.os }}-go-${{ steps.go.outputs.go-version }}- + - name: Set and Echo GOPATH and GOBIN + run: | + $GOPATH = go env GOPATH + $GOBIN = "$GOPATH\bin" + echo "GOPATH 1 is set to $GOPATH" + echo "GOBIN 1 is set to $GOBIN" + echo "GOPATH=$GOPATH" | Out-File -Append -FilePath $env:GITHUB_ENV + echo "GOBIN=$GOBIN" | Out-File -Append -FilePath $env:GITHUB_ENV + echo "GOPATH 2 is set to $GOPATH" + echo "GOBIN 2 is set to $GOBIN" + shell: pwsh + + - name: Echo Go installation path + run: | + echo "Go installation path: $(go env GOROOT)" + ls "$(go env GOROOT)" + ls "$(go env GOROOT)\bin" + shell: bash + + - name: Echo GOPATH and GOBIN + run: | + echo "GOPATH=$env:GOPATH" + echo "GOBIN=$env:GOBIN" + echo "Contents of GOPATH:" + Get-ChildItem -Path $env:GOPATH + echo "Contents of GOBIN:" + Get-ChildItem -Path $env:GOBIN + shell: pwsh + - name: Install MSYS2 run: choco install msys2 shell: powershell @@ -49,10 +78,13 @@ jobs: run: C:\tools\msys64\usr\bin\bash -lc "echo 'Hello from MSYS2'" shell: powershell - - name: Install Make in MSYS2 + - name: Install Make and Unzip in MSYS2 run: | - C:\tools\msys64\usr\bin\bash -lc "pacman -S make --noconfirm" - C:\tools\msys64\usr\bin\bash -lc "pacman -S unzip --noconfirm" + C:\tools\msys64\usr\bin\bash -lc "pacman -S make unzip --noconfirm" + C:\tools\msys64\usr\bin\bash -lc "pacman -S make wget --noconfirm" + C:\tools\msys64\usr\bin\bash -lc "pacman -S make bzip2 --noconfirm" + C:\tools\msys64\usr\bin\bash -lc "wget --version" + C:\tools\msys64\usr\bin\bash -lc "bzip2 --version" shell: powershell - name: Run Unix Commands in MSYS2 @@ -60,19 +92,28 @@ jobs: C:\tools\msys64\usr\bin\bash -lc "echo 'Hello from MSYS2'" C:\tools\msys64\usr\bin\bash -lc "ls" C:\tools\msys64\usr\bin\bash -lc "pwd" - C:\tools\msys64\usr\bin\bash -lc "go version" + C:\tools\msys64\usr\bin\bash -lc "wget --version" + C:\tools\msys64\usr\bin\bash -lc "bzip2 --version" + $WGET = $(Get-Command wget).Source + $BZIP2 = $(Get-Command bzip2).Source + echo "WGET_PATH=$WGET" >> $env:GITHUB_ENV + echo "BZIP2_PATH=$BZIP2" >> $env:GITHUB_ENV + echo "WGET_PATH: $WGET" + echo "BZIP2_PATH: $BZIP2" + $msysBin = "C:\tools\msys64\usr\bin" + echo "MSYS_BIN=$msysBin" >> $env:GITHUB_ENV + echo "PATH info: $env:PATH" + echo "PATH info2: $PATH" shell: powershell - - name: Setup dependencies + - name: Build CLI Core run: | - topLevel=$(git rev-parse --show-toplevel) - echo "Top level: $topLevel" + make build + shell: powershell - # Set up PATH to include Go binaries - export PATH=$PATH:/c/go/bin:$(go env GOPATH)/bin - - C:/tools/msys64/usr/bin/bash -lc "cd '$topLevel' && make tools" + - name: E2E Tests + run: | + C:\tools\msys64\usr\bin\bash -lc "wget --version" + C:\tools\msys64\usr\bin\bash -lc "bzip2 --version" + make e2e-cli-core shell: powershell - env: - MSYS2_PATH_TYPE: inherit # This might help inheriting the PATH correctly - diff --git a/Makefile b/Makefile index ed8bb5862..755884bc7 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,12 @@ include ./plugin-tooling.mk include ./test/e2e/Makefile # Ensure Make is run with bash shell as some syntax below is bash-specific -SHELL := /usr/bin/env bash ROOT_DIR := $(shell git rev-parse --show-toplevel) +ROOT_DIR := $(subst \,/,$(ROOT_DIR)) ARTIFACTS_DIR ?= $(ROOT_DIR)/artifacts +HOME := $(subst \,/,$(HOME)) XDG_CONFIG_HOME := ${HOME}/.config export XDG_CONFIG_HOME @@ -18,6 +19,34 @@ GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) GOHOSTOS ?= $(shell go env GOHOSTOS) GOHOSTARCH ?= $(shell go env GOHOSTARCH) +HOST_OS=$(shell go env GOOS) + +WGET := $(shell which wget) +BZIP2 := $(shell which bzip2) +TAR := $(shell which tar) + +# Load environment variables from GitHub workflow +ifeq ($(GITHUB_ACTIONS),true) + ifeq ($(GOOS),windows) + # Use the values from GitHub Actions + MSYS_BIN ?= $(shell echo $MSYS_BIN) + WGET = $(MSYS_BIN)/wget + BZIP2 = $(MSYS_BIN)/bzip2 + TAR = $(MSYS_BIN)/tar + GITHUB_INFO := Running in GitHub Actions on Windows + endif +endif + +# Ensure $(GOPATH) uses forward slashes +WGET := $(subst \,/,$(WGET)) +BZIP2 := $(subst \,/,$(BZIP2)) +TAR := $(subst \,/,$(TAR)) + +$(info WGET after update: $(WGET)) +$(info BZIP2 after update: $(BZIP2)) +$(info TAR after update: $(TAR)) +$(info $(GITHUB_INFO)) + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -54,7 +83,10 @@ VENDIR := $(TOOLS_BIN_DIR)/vendir YQ := $(TOOLS_BIN_DIR)/yq #TOOLING_BINARIES := $(GOIMPORTS) $(GOLANGCI_LINT) $(VALE) $(MISSPELL) $(CONTROLLER_GEN) $(IMGPKG) $(KUBECTL) $(KIND) $(GINKGO) $(COSIGN) $(GOJUNITREPORT) -TOOLING_BINARIES := $(GOLANGCI_LINT) +TOOLING_BINARIES := $(GOIMPORTS) $(GOLANGCI_LINT) $(VALE) $(MISSPELL) $(CONTROLLER_GEN) $(IMGPKG) $(KUBECTL) $(KIND) $(GINKGO) $(COSIGN) $(GOJUNITREPORT) +#TOOLING_BINARIES := $(GOIMPORTS) + +#TOOLING_BINARIES := $(GOLANGCI_LINT) # Build and version information @@ -145,17 +177,19 @@ build-cli-%: ##Build the Tanzu Core CLI for a platform @echo build $(OS)-$(ARCH) CLI with version: $(BUILD_VERSION) - @if [ "$(filter $(OS)-$(ARCH),$(ENVS))" = "" ]; then\ - printf "\n\n======================================\n";\ - printf "! $(OS)-$(ARCH) is not an officially supported platform!\n";\ - printf "======================================\n\n";\ - fi + @mkdir -p artifacts/$(OS)/$(ARCH)/cli/core/$(BUILD_VERSION) - @if [ "$(OS)" = "windows" ]; then \ - GOOS=$(OS) GOARCH=$(ARCH) $(GO) build -gcflags=all="-l" --ldflags "$(LD_FLAGS)" -o "$(ARTIFACTS_DIR)/$(OS)/$(ARCH)/cli/core/$(BUILD_VERSION)/tanzu-cli-$(OS)_$(ARCH).exe" ./cmd/tanzu/main.go;\ - else \ - GOOS=$(OS) GOARCH=$(ARCH) $(GO) build -gcflags=all="-l" --ldflags "$(LD_FLAGS)" -o "$(ARTIFACTS_DIR)/$(OS)/$(ARCH)/cli/core/$(BUILD_VERSION)/tanzu-cli-$(OS)_$(ARCH)" ./cmd/tanzu/main.go;\ - fi + @echo "Listing artifacts directory:" + @ls artifacts + + @echo "BEfore build" + @echo "Windows build" + @pwd + @cd cmd/tanzu + @pwd + @go build -o $(ARTIFACTS_DIR)/$(OS)/$(ARCH)/cli/core/$(BUILD_VERSION)/tanzu-cli-$(OS)_$(ARCH).exe cmd/tanzu/main.go + @ls cmd/tanzu + @ls $(ARTIFACTS_DIR)/$(OS)/$(ARCH)/cli/core/$(BUILD_VERSION) ## -------------------------------------- ## Plugins-specific @@ -255,28 +289,54 @@ test-with-summary-report: tools .PHONY: e2e-cli-core ## Execute all CLI Core E2E Tests e2e-cli-core: tools crd-package-for-test start-test-central-repo start-airgapped-local-registry e2e-cli-core-all ## Execute all CLI Core E2E Tests + .PHONY: setup-custom-cert-for-test-central-repo setup-custom-cert-for-test-central-repo: ## Setup up the custom ca cert for test-central-repo in the config file - @if [ ! -d $(ROOT_DIR)/hack/central-repo/certs ]; then \ - wget https://storage.googleapis.com/tanzu-cli/data/testcerts/local-central-repo-testcontent.bz2 -O $(ROOT_DIR)/hack/central-repo/local-central-repo-testcontent.bz2;\ - tar xjf $(ROOT_DIR)/hack/central-repo/local-central-repo-testcontent.bz2 -C $(ROOT_DIR)/hack/central-repo/;\ - fi + @echo "ROOT_DIR inside setup-custom-cert-for-test-central-repo: $(ROOT_DIR)" + @echo "WGET inside setup-custom-cert-for-test-central-repo: $(WGET)" + @echo "BZIP2 inside setup-custom-cert-for-test-central-repo: $(BZIP2)" + $(WGET) https://storage.googleapis.com/tanzu-cli/data/testcerts/local-central-repo-testcontent.bz2 -O hack/central-repo/local-central-repo-testcontent.bz2 + $(TAR) xjf hack/central-repo/local-central-repo-testcontent.bz2 -C hack/central-repo/ + @echo "Listing the contents of the hack/central-repo" + @ls -l hack/central-repo + @echo "current directory: $(PWD) files: $(ls)" + @ls echo "Adding docker test central repo cert to the config file" TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" TANZU_CLI_EULA_PROMPT_ANSWER="Yes" $(ROOT_DIR)/bin/tanzu config cert delete localhost:9876 || true $(ROOT_DIR)/bin/tanzu config cert add --host localhost:9876 --ca-cert $(ROOT_DIR)/hack/central-repo/certs/localhost.crt .PHONY: start-test-central-repo +start-test-central-repo: stop-test-central-repo setup-custom-cert-for-test-central-repo ## Starts up a test central repository locally with docker + @if [ ! -d $(ROOT_DIR)/hack/central-repo/registry-content ]; then \ + (cd $(ROOT_DIR)/hack/central-repo && $(TAR) xjf registry-content.bz2 || true;) \ + fi + @echo "Starting docker test central repo" + + @docker run --rm -d -p 9876:443 --name central \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/localhost.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/localhost.key \ + -v "D:\a\tanzu-cli\tanzu-cli\hack\central-repo\registry-content:C:\registry" \ + -v "D:\a\tanzu-cli\tanzu-cli\hack\central-repo\certs:C:\certs" \ + stefanscherer/registry-windows:latest > /dev/null && \ + echo "Started docker test central repo with images:" && \ + $(ROOT_DIR)/hack/central-repo/upload-plugins.sh info + + @echo "Docker test central repo started at localhost:9876" + +.PHONY: start-test-central-repo-11 start-test-central-repo: stop-test-central-repo setup-custom-cert-for-test-central-repo ## Starts up a test central repository locally with docker @if [ ! -d $(ROOT_DIR)/hack/central-repo/registry-content ]; then \ (cd $(ROOT_DIR)/hack/central-repo && tar xjf registry-content.bz2 || true;) \ fi + @echo "Starting docker test central repo" @docker run --rm -d -p 9876:443 --name central \ -v $(ROOT_DIR)/hack/central-repo/certs:/certs \ -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \ -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/localhost.crt \ -e REGISTRY_HTTP_TLS_KEY=/certs/localhost.key \ -v $(ROOT_DIR)/hack/central-repo/registry-content:/var/lib/registry \ - $(REGISTRY_IMAGE) > /dev/null && \ + stefanscherer/registry-windows:latest > /dev/null && \ echo "Started docker test central repo with images:" && \ $(ROOT_DIR)/hack/central-repo/upload-plugins.sh info @@ -285,7 +345,24 @@ stop-test-central-repo: ## Stops and removes the local test central repository @docker container stop central > /dev/null 2>&1 && echo "Stopped docker test central repo" || true .PHONY: start-airgapped-local-registry -start-airgapped-local-registry: stop-airgapped-local-registry +start-airgapped-local-registry-11: stop-airgapped-local-registry + @docker run --rm -d -p 6001:5000 --name temp-airgapped-local-registry \ + $(REGISTRY_IMAGE) > /dev/null && \ + echo "Started docker test airgapped repo at 'localhost:6001'." + + @mkdir -p $(ROOT_DIR)/hack/central-repo/auth && docker run --entrypoint htpasswd httpd:2 -Bbn ${TANZU_CLI_E2E_AIRGAPPED_REPO_WITH_AUTH_USERNAME} ${TANZU_CLI_E2E_AIRGAPPED_REPO_WITH_AUTH_PASSWORD} > $(ROOT_DIR)/hack/central-repo/auth/htpasswd + @docker run --rm -d -p 6002:5000 --name temp-airgapped-local-registry-with-auth \ + -v "D:\a\tanzu-cli\tanzu-cli\hack\central-repo\auth:C:\auth" \ + -e "REGISTRY_AUTH=htpasswd" \ + -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + stefanscherer/registry-windows:latest > /dev/null && \ + echo "Started docker test airgapped repo with authentication at 'localhost:6002'." + + @docker logout localhost:6002 || true + +.PHONY: start-airgapped-local-registry-11 +start-airgapped-local-registry-11: stop-airgapped-local-registry @docker run --rm -d -p 6001:5000 --name temp-airgapped-local-registry \ $(REGISTRY_IMAGE) > /dev/null && \ echo "Started docker test airgapped repo at 'localhost:6001'." @@ -301,6 +378,7 @@ start-airgapped-local-registry: stop-airgapped-local-registry @docker logout localhost:6002 || true + .PHONY: stop-airgapped-local-registry stop-airgapped-local-registry: @docker stop temp-airgapped-local-registry temp-airgapped-local-registry-with-auth > /dev/null 2>&1 && \ diff --git a/hack/tools/Makefile b/hack/tools/Makefile index 5bcdaf1a0..788d72e3e 100644 --- a/hack/tools/Makefile +++ b/hack/tools/Makefile @@ -62,12 +62,51 @@ VAL_URL := https://github.com/errata-ai/vale/releases/download/v$(VALE_VERSION)/ sed 's,darwin,macOS,g'| sed 's,linux,Linux,g'| sed 's,windows,Windows,g')_$(shell echo $(HOST_ARCH) | sed 's,amd64,64-bit,g')$(shell echo $(HOST_OS) | \ sed 's,darwin,.tar.gz,g'| sed 's,linux,.tar.gz,g'| sed 's,windows,.zip,g') -ifeq ($(CI),true) -GOBIN := $(subst /,\,$(ROOT_DIR)/hack/tools/$(BIN_DIR)) + +ifeq ($(GITHUB_ACTIONS),true) + # Use the values from GitHub Actions + GOPATH := $(shell echo $$GOPATH) + GOBIN := $(shell echo $$GOBIN) + GITHUB_INFO := Running in GitHub Actions else -GOBIN := $(ROOT_DIR)/hack/tools/$(BIN_DIR) + # Use the system's GOPATH and GOBIN; if they're not set, use defaults + GOPATH ?= $(HOME)/go + GOBIN ?= $(GOPATH)/bin + GITHUB_INFO := Not running in GitHub Actions endif +$(info GOPATH before update: $(GOPATH)) +$(info GOBIN before update: $(GOBIN)) +$(info $(GITHUB_INFO)) + +# Normalize $(ROOT_DIR) and ensure no double slashes before appending +GOBIN := $(patsubst %/,%,$(subst \,/,${ROOT_DIR}))/hack/tools/$(BIN_DIR) + +# Ensure $(GOPATH) uses forward slashes +GOPATH := $(subst \,/,$(GOPATH)) + + +# It's best to use these echo commands in an actual recipe, not directly in the conditional logic +# For demonstration purposes, they are kept here but consider moving them into a rule's command +$(info GOBIN after update: $(GOBIN)) +$(info GOPATH after update: $(GOPATH)) + + +golangci-lint: $(GOLANGCI_LINT) ## Install golangci-lint +$(GOLANGCI_LINT): + @echo "GOPATH current: $(GOPATH)" + @echo "GOBIN current: $(GOBIN)" + mkdir -p $(BIN_DIR) + @ls $(GOPATH) + @ls $(GOPATH)/bin + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(GOLANGCI_LINT_VERSION) + @ls $(GOPATH) + @ls $(GOPATH)/bin + @echo "GOBIN files: $(GOBIN)" + @ls $(GOBIN) +#mv $(GOPATH)/$(GOLANGCI_LINT) $(GOBIN) && chmod a+x $(GOBIN)/$(notdir $(GOLANGCI_LINT)) + + ## -------------------------------------- ## Help ## -------------------------------------- @@ -81,11 +120,11 @@ help: ## Display this help goimports: $(GOIMPORTS) ## Install goimports $(GOIMPORTS): - @echo "BIN_DIR: $(BIN_DIR)" - @echo "ROOT_DIR: $(ROOT_DIR)" - @echo "GOBIN path: $(ROOT_DIR)/hack/tools/$(BIN_DIR)" - @mkdir -p $(BIN_DIR) - @GOBIN=$(subst \,/,$(ROOT_DIR)/hack/tools/$(BIN_DIR)) go install golang.org/x/tools/cmd/goimports@v$(GOIMPORTS_VERSION) + @echo "gobin in goimports: $(GOBIN)" + mkdir -p $(BIN_DIR) + go install golang.org/x/tools/cmd/goimports@v$(GOIMPORTS_VERSION) + @echo "GOBIN files: $(GOBIN)" + @ls $(GOBIN) #mkdir -p $(BIN_DIR) #GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install golang.org/x/tools/cmd/goimports@v$(GOIMPORTS_VERSION) @@ -95,13 +134,6 @@ $(GOIMPORTS): #mv $(GOPATH)/bin/goimports $(GOIMPORTS) #GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install golang.org/x/tools/cmd/goimports@v$(GOIMPORTS_VERSION) -golangci-lint: $(GOLANGCI_LINT) ## Install golangci-lint -$(GOLANGCI_LINT): - @echo "Installing golangci-lint to $(GOLANGCI_LINT)" - @echo "GOBIN path: $(ROOT_DIR)/hack/tools/$(BIN_DIR)" - @echo "GOBIN will be set to $(GOBIN)" - mkdir -p $(BIN_DIR) - GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(GOLANGCI_LINT_VERSION) vale: $(VALE) $(VALE): @@ -123,18 +155,17 @@ $(VALE): controller-gen: $(CONTROLLER_GEN) ## Build controller-gen $(CONTROLLER_GEN): mkdir -p $(BIN_DIR) - #go build -tags=tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen - GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install sigs.k8s.io/controller-tools/cmd/controller-gen@v$(CONTROLLER_TOOLS_VERSION) + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v$(CONTROLLER_TOOLS_VERSION) cosign: $(COSIGN) ## Install cosign $(COSIGN): mkdir -p $(BIN_DIR) - GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install github.com/sigstore/cosign/v2/cmd/cosign@v$(COSIGN_VERSION) + go install github.com/sigstore/cosign/v2/cmd/cosign@v$(COSIGN_VERSION) misspell: $(MISSPELL) ## Install misspell $(MISSPELL): mkdir -p $(BIN_DIR) - GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install github.com/client9/misspell/cmd/misspell@v$(MISSPELL_VERSION) + go install github.com/client9/misspell/cmd/misspell@v$(MISSPELL_VERSION) imgpkg: $(IMGPKG) ## Install imgpkg $(IMGPKG): @@ -198,12 +229,13 @@ $(KIND): ginkgo: $(GINKGO) ## Install ginkgo $(GINKGO): mkdir -p $(BIN_DIR) - GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION) + go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION) go-junit-report: $(GOJUNITREPORT) ## Install go-junit-report $(GOJUNITREPORT): mkdir -p $(BIN_DIR) - GOBIN=$(ROOT_DIR)/hack/tools/$(BIN_DIR) go install github.com/jstemmer/go-junit-report@latest + go install github.com/jstemmer/go-junit-report@latest + @ls $(GOBIN) ## -------------------------------------- ## Cleanup