From ed2285cb18520fc530224816bef038f1b1cbde4e Mon Sep 17 00:00:00 2001 From: Shahzad Lone Date: Tue, 23 Jan 2024 15:43:52 -0500 Subject: [PATCH] ci(i): Add YAML linter (#2241) ## Relevant issue(s) - Resolves #2245 ## Description - Basic YAML linter Action - Fix all YAML linter errors - PR title change when using the manual action to combine dependabot PRs - Update default options of the combine action according to our common usage ## How has this been tested? - CI checks for this now Specify the platform(s) on which this was tested: - WSL (Manjaro) --- .github/dependabot.yml | 2 +- .github/workflows/build-then-deploy-ami.yml | 2 +- .github/workflows/combine-bot-prs.yml | 35 ++++----- .github/workflows/lint-then-benchmark.yml | 3 +- .github/workflows/lint.yml | 22 ++++-- .../preview-ami-with-terraform-plan.yml | 2 +- .../workflows/test-and-upload-coverage.yml | 2 +- .github/workflows/validate-containerfile.yml | 1 - .goreleaser.yaml | 67 ++++++++++------- Makefile | 72 ++++++++++++++++--- tools/cloud/akash/deploy.yaml | 6 +- tools/configs/yamllint.yaml | 39 ++++++++++ 12 files changed, 186 insertions(+), 67 deletions(-) create mode 100644 tools/configs/yamllint.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e48d4303f1..577198d399 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,4 +15,4 @@ updates: labels: - "dependencies" commit-message: - prefix: "bot" \ No newline at end of file + prefix: "bot" diff --git a/.github/workflows/build-then-deploy-ami.yml b/.github/workflows/build-then-deploy-ami.yml index ce6be1e0bd..4423d70659 100644 --- a/.github/workflows/build-then-deploy-ami.yml +++ b/.github/workflows/build-then-deploy-ami.yml @@ -109,7 +109,7 @@ jobs: - name: Terraform validation run: terraform validate -no-color - - name: List workspaces + - name: List workspaces run: ls workspaces - name: Terraform Apply diff --git a/.github/workflows/combine-bot-prs.yml b/.github/workflows/combine-bot-prs.yml index 59a7910ced..abe21143d8 100644 --- a/.github/workflows/combine-bot-prs.yml +++ b/.github/workflows/combine-bot-prs.yml @@ -22,10 +22,10 @@ on: default: 'dependabot' mustBeGreen: - description: 'Only combine PRs that are green (status is success). Set to false if repo does not run checks' + description: 'Only combine PRs that are green (status is success). Keep false if repo does not run checks' type: boolean required: true - default: true + default: false combineBranchName: description: 'Name of the branch to combine PRs into' @@ -35,7 +35,7 @@ on: ignoreLabel: description: 'Exclude PRs with this label' required: true - default: 'nocombine' + default: 'DO NOT MERGE' jobs: combine-bot-prs: @@ -44,12 +44,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v6 + - name: Set current date as env variable + run: echo "CURRENT_DATE=$(date +'%d-%m-%Y')" >> ${GITHUB_ENV} + - name: Create combined pr id: create-combined-pr - - name: Create combined pr - + uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -66,7 +66,7 @@ jobs: if (branch.startsWith('${{ github.event.inputs.branchPrefix }}')) { console.log('Branch matched prefix: ' + branch); let statusOK = true; - if(${{ github.event.inputs.mustBeGreen }}) { + if (${{ github.event.inputs.mustBeGreen }}) { console.log('Checking green status: ' + branch); const stateQuery = `query($owner: String!, $repo: String!, $pull_number: Int!) { repository(owner: $owner, name: $repo) { @@ -92,17 +92,17 @@ jobs: const [{ commit }] = result.repository.pullRequest.commits.nodes; const state = commit.statusCheckRollup.state console.log('Validating status: ' + state); - if(state != 'SUCCESS') { + if (state != 'SUCCESS') { console.log('Discarding ' + branch + ' with status ' + state); statusOK = false; } } console.log('Checking labels: ' + branch); const labels = pull['labels']; - for(const label of labels) { + for (const label of labels) { const labelName = label['name']; console.log('Checking label: ' + labelName); - if(labelName == '${{ github.event.inputs.ignoreLabel }}') { + if (labelName == '${{ github.event.inputs.ignoreLabel }}') { console.log('Discarding ' + branch + ' with label ' + labelName); statusOK = false; } @@ -110,7 +110,10 @@ jobs: if (statusOK) { console.log('Adding branch to array: ' + branch); const prString = '#' + pull['number'] + ' ' + pull['title']; - branchesAndPRStrings.push({ branch, prString }); + branchesAndPRStrings.push({ + branch, + prString + }); baseBranch = pull['base']['ref']; baseBranchSHA = pull['base']['sha']; } @@ -135,7 +138,7 @@ jobs: let combinedPRs = []; let mergeFailedPRs = []; - for(const { branch, prString } of branchesAndPRStrings) { + for (const { branch, prString } of branchesAndPRStrings) { try { await github.rest.repos.merge({ owner: context.repo.owner, @@ -153,15 +156,15 @@ jobs: console.log('Creating combined PR'); const combinedPRsString = combinedPRs.join('\n'); - let body = '✅ This PR was created by the Combine PRs action by combining the following PRs:\n' + combinedPRsString; - if(mergeFailedPRs.length > 0) { + let body = '✅ This PR was created by combining the following PRs:\n' + combinedPRsString; + if (mergeFailedPRs.length > 0) { const mergeFailedPRsString = mergeFailedPRs.join('\n'); body += '\n\n⚠️ The following PRs were left out due to merge conflicts:\n' + mergeFailedPRsString } await github.rest.pulls.create({ owner: context.repo.owner, repo: context.repo.repo, - title: 'bot: Combined PRs', + title: 'bot: Update dependencies (bulk dependabot PRs) ${CURRENT_DATE}', head: '${{ github.event.inputs.combineBranchName }}', base: baseBranch, body: body diff --git a/.github/workflows/lint-then-benchmark.yml b/.github/workflows/lint-then-benchmark.yml index a251e24fa5..c984cce3ef 100644 --- a/.github/workflows/lint-then-benchmark.yml +++ b/.github/workflows/lint-then-benchmark.yml @@ -270,7 +270,8 @@ jobs: github.event_name == 'pull_request' && github.base_ref == 'develop' run: > - ${GOPATH}/bin/benchstat -html -alpha 1.1 develop.txt current.txt | sed -n "//,/<\/body>/p" > comparison.html && + ${GOPATH}/bin/benchstat -html -alpha 1.1 develop.txt current.txt | + sed -n "//,/<\/body>/p" > comparison.html && ./tools/scripts/pretty-benchstat-html.sh comparison.html > pretty-comparison.md - name: Comment Benchmark Results on PR diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e7f8b6f3ee..b482903cad 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,8 +24,8 @@ permissions: contents: read jobs: - lint: - name: Lint job + lint-go: + name: Lint GoLang job runs-on: ubuntu-latest @@ -39,9 +39,8 @@ jobs: go-version: "1.21" check-latest: true - - name: Check linting through golangci-lint + - name: Run golangci-lint linter uses: golangci/golangci-lint-action@v3 - with: # Required: the version of golangci-lint is required. # Note: The version should not pick the patch version as the latest patch @@ -68,3 +67,18 @@ jobs: # anyways so there shouldn't be any linter errors anyways. The enforces us to # always have a clean lint state. only-new-issues: false + + lint-yaml: + name: Lint YAML job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Run yamllint linter + uses: ibiqlik/action-yamllint@v3 + with: + config_file: tools/configs/yamllint.yaml + file_or_dir: . diff --git a/.github/workflows/preview-ami-with-terraform-plan.yml b/.github/workflows/preview-ami-with-terraform-plan.yml index ed2fef6f0c..25e975a247 100644 --- a/.github/workflows/preview-ami-with-terraform-plan.yml +++ b/.github/workflows/preview-ami-with-terraform-plan.yml @@ -131,5 +131,5 @@ jobs: if: steps.terraform-plan.outcome == 'failure' run: exit 1 - - name: List workspaces + - name: List workspaces run: ls workspaces diff --git a/.github/workflows/test-and-upload-coverage.yml b/.github/workflows/test-and-upload-coverage.yml index 38bcedd95c..60858b1f86 100644 --- a/.github/workflows/test-and-upload-coverage.yml +++ b/.github/workflows/test-and-upload-coverage.yml @@ -105,7 +105,7 @@ jobs: needs: run-tests - # Important to know: + # Important to know: # - We didn't use `if: always()` here, so this job doesn't run if we manually canceled. # - `if: success()` is always implied unless `always()` or `failure()` is specified. if: success() || failure() diff --git a/.github/workflows/validate-containerfile.yml b/.github/workflows/validate-containerfile.yml index b3315861ad..260e0dba89 100644 --- a/.github/workflows/validate-containerfile.yml +++ b/.github/workflows/validate-containerfile.yml @@ -54,4 +54,3 @@ jobs: - name: Test Docker image run: docker run --rm ${{ env.TEST_TAG }} - diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 119447d180..7995056d42 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -15,7 +15,7 @@ builds: goarch: - amd64 - arm64 - # A build with the playground included. + # A build with the playground included. - id: "defradb_playground" main: ./cmd/defradb flags: @@ -27,30 +27,44 @@ builds: goarch: - amd64 - arm64 - + partial: by: target archives: - id: defradb_playground - builds: + builds: - defradb_playground format: binary # this name template makes the OS and Arch compatible with the results of `uname`. - name_template: '{{ .Binary }}_playground_{{ .Version }}_{{ .Os }}_{{- if eq .Arch "amd64" }}x86_64{{- else }}{{ .Arch }}{{ end }}{{- if .Arm }}v{{ .Arm }}{{ end }}' + name_template: >- + {{ .Binary }}_playground_{{ .Version }}_{{ .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} - id: defradb - builds: + builds: - defradb format: binary # this name template makes the OS and Arch compatible with the results of `uname`. - name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{- if eq .Arch "amd64" }}x86_64{{- else }}{{ .Arch }}{{ end }}{{- if .Arm }}v{{ .Arm }}{{ end }}' + name_template: >- + {{ .Binary }}_{{ .Version }}_{{ .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} release: target_commitish: '{{ .Commit }}' - header: | - DefraDB v{{ .Major }}.{{ .Minor }} is a major pre-production release. Until the stable version 1.0 is reached, the SemVer minor patch number will denote notable releases, which will give the project freedom to experiment and explore potentially breaking changes. + header: > + DefraDB v{{ .Major }}.{{ .Minor }} is a major pre-production release. + Until the stable version 1.0 is reached, the SemVer minor patch number will denote notable releases, + which will give the project freedom to experiment and explore potentially breaking changes. + + To get a full outline of the changes, we invite you to review the official changelog below. + This release does include a Breaking Change to existing v{{ .Major }}.{{ .Minor }}.x databases. + If you need help migrating an existing deployment, reach out at hello@source.network or join + our Discord at https://discord.gg/w7jYQVJ/. - To get a full outline of the changes, we invite you to review the official changelog below. This release does include a Breaking Change to existing v{{ .Major }}.{{ .Minor }}.x databases. If you need help migrating an existing deployment, reach out at hello@source.network or join our Discord at https://discord.gg/w7jYQVJ/. name_template: "v{{ .Version }} Release" changelog: @@ -85,21 +99,20 @@ milestones: name_template: "DefraDB v{{ .Major }}.{{ .Minor }}" dockers: -- ids: - - "defradb_playground" - image_templates: - - "{{ .Env.GITHUB_REPOSITORY }}:latest" - - "{{ .Env.GITHUB_REPOSITORY }}:{{ .Version }}" - - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .Version }}" - use: buildx - build_flag_templates: - - "--pull" - - "--label=org.opencontainers.image.description=DefraDB is a Peer-to-Peer Edge Database." - - "--label=org.opencontainers.image.created={{ .Date }}" - - "--label=org.opencontainers.image.name={{ .ProjectName }}" - - "--label=org.opencontainers.image.revision={{ .FullCommit }}" - - "--label=org.opencontainers.image.version={{ .Version }}" - - "--label=org.opencontainers.image.source={{ .GitURL }}" - - "--platform=linux/amd64" - dockerfile: ./tools/goreleaser.containerfile - + - ids: + - "defradb_playground" + image_templates: + - "{{ .Env.GITHUB_REPOSITORY }}:latest" + - "{{ .Env.GITHUB_REPOSITORY }}:{{ .Version }}" + - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .Version }}" + use: buildx + build_flag_templates: + - "--pull" + - "--label=org.opencontainers.image.description=DefraDB is a Peer-to-Peer Edge Database." + - "--label=org.opencontainers.image.created={{ .Date }}" + - "--label=org.opencontainers.image.name={{ .ProjectName }}" + - "--label=org.opencontainers.image.revision={{ .FullCommit }}" + - "--label=org.opencontainers.image.version={{ .Version }}" + - "--label=org.opencontainers.image.source={{ .GitURL }}" + - "--platform=linux/amd64" + dockerfile: ./tools/goreleaser.containerfile diff --git a/Makefile b/Makefile index 4e10dc142c..56ff8cbcbe 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,33 @@ ifndef VERBOSE MAKEFLAGS+=--no-print-directory endif +# Detect OS (`Linux`, `Darwin`, `Windows`) +# Note: can use `lsb_release --id --short` for more specfic linux distro information. +OS_GENERAL := Unknown +ifeq ($(OS),Windows_NT) + OS_GENERAL := Windows +else + OS_GENERAL := $(shell sh -c 'uname 2>/dev/null || echo Unknown') +endif + +# Detect OS specfic package manager if possible (`apt`, `yum`, `pacman`, `brew`, `choco`) +OS_PACKAGE_MANAGER := Unknown +ifeq ($(OS_GENERAL),Linux) + ifneq ($(shell which apt 2>/dev/null),) + OS_PACKAGE_MANAGER := apt + else ifneq ($(shell which yum 2>/dev/null),) + OS_PACKAGE_MANAGER := yum + else ifneq ($(shell which pacman 2>/dev/null),) + OS_PACKAGE_MANAGER := pacman + else ifneq ($(shell which dnf 2>/dev/null),) + OS_PACKAGE_MANAGER := dnf + endif +else ifeq ($(OS_GENERAL),Darwin) + OS_PACKAGE_MANAGER := brew +else ifeq ($(OS_GENERAL),Windows) + OS_PACKAGE_MANAGER := choco +endif + # Provide info from git to the version package using linker flags. ifeq (, $(shell which git)) $(error "No git in $(PATH), version information won't be included") @@ -18,6 +45,15 @@ else VERSION_GITRELEASE=$(shell git describe --tags) endif +$(info ----------------------------------------); +$(info OS = $(OS_GENERAL)); +$(info PACKAGE_MANAGER = $(OS_PACKAGE_MANAGER)); +$(info GOINFO = $(VERSION_GOINFO)); +$(info GITCOMMIT = $(VERSION_GITCOMMIT)); +$(info GITCOMMITDATE = $(VERSION_GITCOMMITDATE)); +$(info GITRELEASE = $(VERSION_GITRELEASE)); +$(info ----------------------------------------); + BUILD_FLAGS=-trimpath -ldflags "\ -X 'github.com/sourcenetwork/defradb/version.GoInfo=$(VERSION_GOINFO)'\ -X 'github.com/sourcenetwork/defradb/version.GitRelease=$(VERSION_GITRELEASE)'\ @@ -47,6 +83,15 @@ default: install: @go install $(BUILD_FLAGS) ./cmd/defradb +.PHONY: install\:manpages +install\:manpages: +ifeq ($(OS_GENERAL),Linux) + cp build/man/* /usr/share/man/man1/ +endif +ifneq ($(OS_GENERAL),Linux) + @echo "Direct installation of Defradb's man pages is not supported on your system." +endif + # Usage: # - make build # - make build path="path/to/defradb-binary" @@ -78,9 +123,23 @@ client\:dump: client\:add-schema: ./build/defradb client schema add -f examples/schema/bookauthpub.graphql +.PHONY: deps\:lint-go +deps\:lint-go: + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54 + +.PHONY: deps\:lint-yaml +deps\:lint-yaml: +ifeq (, $(shell which yamllint)) + $(info YAML linter 'yamllint' not found on the system, please install it.) + $(info Can try using your local package manager: $(OS_PACKAGE_MANAGER)) +else + $(info YAML linter 'yamllint' already installed.) +endif + .PHONY: deps\:lint deps\:lint: - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54 + @$(MAKE) deps:lint-go && \ + $(MAKE) deps:lint-yaml .PHONY: deps\:test deps\:test: @@ -300,6 +359,7 @@ validate\:circleci: .PHONY: lint lint: golangci-lint run --config tools/configs/golangci.yaml + yamllint -c tools/configs/yamllint.yaml . .PHONY: lint\:fix lint\:fix: @@ -334,13 +394,3 @@ docs\:manpages: docs\:godoc: godoc -http=:6060 # open http://localhost:6060/pkg/github.com/sourcenetwork/defradb/ - -detectedOS := $(shell uname) -.PHONY: install\:manpages -install\:manpages: -ifeq ($(detectedOS),Linux) - cp build/man/* /usr/share/man/man1/ -endif -ifneq ($(detectedOS),Linux) - @echo "Direct installation of Defradb's man pages is not supported on your system." -endif diff --git a/tools/cloud/akash/deploy.yaml b/tools/cloud/akash/deploy.yaml index c6f7070f98..48d86fff8b 100644 --- a/tools/cloud/akash/deploy.yaml +++ b/tools/cloud/akash/deploy.yaml @@ -5,7 +5,7 @@ services: defradb: image: sourcenetwork/defradb:v0.6.0 args: - - start + - start - --url=0.0.0.0:9181 expose: - port: 9161 @@ -40,7 +40,7 @@ profiles: - "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" - "akash18qa2a2ltfyvkyj0ggj3hkvuj6twzyumuaru9s4" pricing: - defradb: + defradb: denom: uakt amount: 10000 @@ -48,4 +48,4 @@ deployment: defradb: akash: profile: defradb - count: 1 \ No newline at end of file + count: 1 diff --git a/tools/configs/yamllint.yaml b/tools/configs/yamllint.yaml new file mode 100644 index 0000000000..b7f98b844a --- /dev/null +++ b/tools/configs/yamllint.yaml @@ -0,0 +1,39 @@ +# This file contains our linter configurations that will be used for all Source Inc. projects. + +yaml-files: + - '*.yaml' + - '*.yml' + - '.yamllint' + +rules: + # Disabled lint rules + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-values: disable + float-values: disable + key-ordering: disable + octal-values: disable + quoted-strings: disable + + # Enabled lint rules + anchors: enable + braces: enable + brackets: enable + colons: enable + commas: enable + empty-lines: enable + hyphens: enable + key-duplicates: enable + new-line-at-end-of-file: enable + new-lines: enable + trailing-spaces: enable + truthy: disable + line-length: + max: 120 + level: error + indentation: + indent-sequences: consistent + spaces: consistent + check-multi-line-strings: false