From 38e953d6a0a3bd0ce0f4bacb6fd5c61eca315d8d Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Fri, 22 Sep 2023 11:54:07 +0200 Subject: [PATCH] test: add e2e tests for windows and linux - add COLLIE_VERSION env var support to install scripts - fix a typo in collie compliance tree warning --- .gitattributes | 6 +++ .github/workflows/e2e.yml | 68 +++++++++++++++++++++++++ .github/workflows/release.yml | 2 +- CONTRIBUTING.md | 10 ++-- install.ps1 | 19 ++++--- install.sh | 15 ++++-- src/commands/compliance/tree.command.ts | 2 +- test/e2e.ps1 | 61 ++++++++++++++++++++++ test/e2e.sh | 56 ++++++++++++++++++++ 9 files changed, 224 insertions(+), 15 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/e2e.yml create mode 100755 test/e2e.ps1 create mode 100755 test/e2e.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..ff38a9be --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Force LF file endings for all text files +* text eol=lf + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary \ No newline at end of file diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..02620dde --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,68 @@ +name: e2e +on: + # temporary, just for testing the workflow + push: + branches: [ feature/e2e-tests ] + # run after the "release" workflow https://stackoverflow.com/a/64733705/125407 + workflow_run: + workflows: ["release"] + branches: [main] + types: + - completed + +# TODO: replace "feature/e2e-tests" with "main" before merging to main branch! + +jobs: + # this test is super basic and just checks whether collie blows up + e2e-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: install dependencies + uses: crazy-max/ghaction-chocolatey@v3 + # note: all other dependencies are installed in the image already + with: + args: install --force terraform terragrunt terraform-docs + - name: install collie + shell: pwsh + env: + COLLIE_VERSION: ${{ github.event.release.tag_name }} + run: | + irm https://raw.githubusercontent.com/meshcloud/collie-cli/feature/e2e-tests/install.ps1 | iex + + # github doesn't allow adding to PATH any other way, see https://stackoverflow.com/a/71579543/125407 + Add-Content $env:GITHUB_PATH "C:\Users\runneradmin\collie-cli" + - name: collie info + shell: pwsh + run: | + collie info + - name: test + shell: pwsh + run: .\test\e2e.ps1 + e2e-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: install dependencies + # note: we forego default.nix and to be as close to a realistic user experience as possible + # all the other dependencies are already present on the runner images, so we use those + # and just install the missing ones via nix + uses: cachix/install-nix-action@v18 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - uses: rrbutani/use-nix-shell-action@v1 + with: + packages: terragrunt terraform-docs + - name: install collie + env: + COLLIE_VERSION: ${{ github.event.release.tag_name }} + run: | + curl -sf -L https://raw.githubusercontent.com/meshcloud/collie-cli/main/install.sh | sudo bash + - name: collie info + run: | + collie info + - name: test + run: | + ./test/e2e.sh \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5a4272f..8f15192e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: release on: release: - types: [published] + types: [prereleased] jobs: create-release: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79fff723..7b511289 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,16 +41,20 @@ The versions have this format `vMAJOR.MINOR.PATCH`, e.g. `v0.1.0`. The release is triggered by [creating a new release via GitHub's UI](https://github.com/meshcloud/collie-cli/releases/new) -as this gives us a good workflow to review and edit release notes before -publishing. +**as a prerelease**. This gives us a good workflow to review and edit release +notes before publishing as well as running end to end tests before publishing +the release. The full workflow from forth to back is as follows: - decide on the version number you want to use for the release choose a new tag "vMAJOR.MINOR.PATCH" -- click "Generate release notes" +- click "Generate release notes" and tick * - the release binaries are created automatically by the [release workflow](.github/workflows/releases.yml) +- the e2e tests are run after the release workflow completes + [e2e workflow](.github/workflows/e2e.yml) +- manually inspect the e2e test outcomes and publish the release > Note: The install script also always works with thse latest version, so you're > done. Nice! 🎉 diff --git a/install.ps1 b/install.ps1 index f10b0c0e..1b9f9f1a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1,15 +1,21 @@ -Clear-Host Write-Host "`nStarting Installation of Collie ..." -ForegroundColor Green $ProgressPreference = "SilentlyContinue" # Don't remove this! Otherwise it will slow down the Download-Process under PS 5 # Check the latest release to get the exact download URL try { - $query = Invoke-RestMethod -Uri "https://api.github.com/repos/meshcloud/collie-cli/releases/latest" + $uri = "https://api.github.com/repos/meshcloud/collie-cli/releases/latest" + if (![string]::IsNullOrEmpty($env:COLLIE_VERSION )) { + $uri = "https://api.github.com/repos/meshcloud/collie-cli/releases/tags/$($env:COLLIE_VERSION)" + } + + Write-Host "Searching for collie release at $uri" + $query = Invoke-RestMethod -Uri $uri $downloadUrl = $query.assets | Where-Object { $_.browser_download_url -match "windows-msvc.exe" } } catch { - throw "[ERROR] while downloading installation file. Check your Internet Connection: $(_.Exception.Message)!" + Write-Host "[ERROR] while locating collie release!" + throw } $installationPath = "${HOME}\collie-cli" @@ -23,7 +29,7 @@ if ( !(Test-Path -Path $installationExec) ) { } elseif ( Test-Path -Path $installationExec) { $replace = $(Write-Host "Collie already exists. Do you want to replace the existing Installation (y/n)? " -NoNewLine -ForegroundColor Green; Read-Host) - if ($replace -like "y") { + if ($replace -eq "y") { Remove-Item $installationExec -Force } else { Exit } @@ -36,7 +42,8 @@ try { if ($?) { Write-Host "[OK] Download-Size '$([math]::Round($download.RawContentLength/1MB)),2 MB'`n" -ForegroundColor Green } } catch { - throw "[ERROR] while downloading Installation File: $($($_.Exception).Message)!" + Write-Host "[ERROR] while downloading installation file!" + throw } # Check if Collie is already in PATH @@ -47,7 +54,7 @@ if (!(";$Path;".ToLower() -like "*;$BinDir;*".ToLower())) { # Ask user whether to add Collie to PATH $userenv = $(Write-Host "Could not find collie CLI on PATH. Do you want to add collie CLI to PATH? (y/n)" -NoNewLine -ForegroundColor Green; Read-Host) - if ($userenv -like "y") { + if ($userenv -eq "y") { Write-Host "Adding $BinDir to PATH" -ForegroundColor Green [Environment]::SetEnvironmentVariable('Path', "$Path;$BinDir", $User) $Env:Path += ";$BinDir" diff --git a/install.sh b/install.sh index 0c352253..5e243481 100755 --- a/install.sh +++ b/install.sh @@ -21,15 +21,22 @@ for i in $REQ_COMMANDS do command -v "$i" >/dev/null && continue || { echo "$i command not found. You need to install if before running this script"; exit 1; } done + +COLLIE_VERSION=${COLLIE_VERSION:-} +release_url="https://api.github.com/repos/meshcloud/collie-cli/releases/latest" +if [ ! -z "${COLLIE_VERSION}" ]; then + release_url="https://api.github.com/repos/meshcloud/collie-cli/releases/tags/${COLLIE_VERSION}" +fi + # Download artifacts based on CPU architecture and OS type case "$(uname -s)" in Darwin) - url=$(curl -s https://api.github.com/repos/meshcloud/collie-cli/releases/latest | grep "browser_download_url.*$arch.*apple" | cut -d : -f 2,3 | tr -d \" | tr -d \ ) + artifact_url=$(curl -s "$release_url" | grep "browser_download_url.*$arch.*apple" | cut -d : -f 2,3 | tr -d \" | tr -d \ ) name="collie-$arch-apple-darwin" ;; Linux) echo "detected Linux" - url=$(curl -s https://api.github.com/repos/meshcloud/collie-cli/releases/latest | grep "browser_download_url.*linux" | cut -d : -f 2,3 | tr -d \" | tr -d \ ) + artifact_url=$(curl -s "$release_url" | grep "browser_download_url.*linux" | cut -d : -f 2,3 | tr -d \" | tr -d \ ) name="collie-x86_64-unknown-linux-gnu" ;; CYGWIN*|MINGW32*|MSYS*|MINGW*) @@ -42,8 +49,8 @@ case "$(uname -s)" in ;; esac -echo "Downloading the artifact... (${url})" -curl "${url}" -L -o collie.tar.gz --silent +echo "Downloading the artifact... (${artifact_url})" +curl "${artifact_url}" -L -o collie.tar.gz --silent # untar and movin the artifact echo "Unpacking the artifact and copying the binary..." tar -zxf collie.tar.gz -C /usr/local/bin diff --git a/src/commands/compliance/tree.command.ts b/src/commands/compliance/tree.command.ts index b101260d..a5d07984 100644 --- a/src/commands/compliance/tree.command.ts +++ b/src/commands/compliance/tree.command.ts @@ -50,7 +50,7 @@ export function registerTreeCmd(program: TopLevelCommand) { ); if (!hasAnyApplicableStatements) { - logger.warn("no compliance control statements found in an< kit module"); + logger.warn("no compliance control statements found in any kit module"); logger.tip( `Add a compliance section to your kit module frontmatter like this\n` + statementExample, diff --git a/test/e2e.ps1 b/test/e2e.ps1 new file mode 100755 index 00000000..5ab646be --- /dev/null +++ b/test/e2e.ps1 @@ -0,0 +1,61 @@ +$tmpdir=New-TemporaryFile | %{ rm $_; mkdir $_ } +echo "Running test in $tmpdir - please remember to cleanup" + +cd $tmpdir + +# don't ask..., if you must - https://stackoverflow.com/questions/5596982/using-powershell-to-write-a-file-in-utf-8-without-the-bom#comment79966549_5596984 +[System.Environment]::CurrentDirectory = (Get-Location).Path + +collie init . + +echo "Testing foundation features" +collie foundation new f + +# fake a platform config file + +# equivalent of mkdir -p +md foundations/f/platforms/az/ -ea 0 + +# powershell needs some convicing to write files at UTF8 without BOM... this is the only thing that works +$platform_md=@" +--- +id: az +name: Likvid Bank Azure +azure: + aadTenantId: 1234-1234 + subscriptionId: 1234-1234 +cli: + az: {} +--- +# Likvid Bank Azure +Test test +"@ +[System.IO.File]::WriteAllLines("foundations/f/platforms/az/README.md", $platform_md) + + +echo "Testing kit features" +collie kit import foundation/docs +collie kit import azure/bootstrap +collie kit apply azure/bootstrap --foundation f --platform az + +collie kit new azure/dummy dummy + +$variables_tf=@" +variable "my_input_var" {} +"@ +[IO.File]::WriteAllLines("kit/azure/dummy/variables.tf", $variables_tf) + +collie kit apply azure/dummy --foundation f --platform az + +collie foundation deploy --auto-approve + +echo "Testing compliance features" +collie compliance import cfmm + +echo "Testing outputs" +collie foundation tree +collie kit tree +collie compliance tree + +echo "Testing documentation" +collie foundation docs --build diff --git a/test/e2e.sh b/test/e2e.sh new file mode 100755 index 00000000..907e910c --- /dev/null +++ b/test/e2e.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -o errexit # exit on error +set -o errtrace # enables ERR traps so we can run cleanup +set -o pipefail # exit on error in a pipe, without this only the status of the last command in a pipe is considered +set -o nounset # exit on undefined variables + +tmpdir=$(mktemp -d) +echo "Running test in $tmpdir - please remember to cleanup" + +cd "$tmpdir" + +collie init . + +echo "Testing foundation features" +collie foundation new f > /dev/null + +# fake a platform config file +mkdir -p ./foundations/f/platforms/az/ +cat > ./foundations/f/platforms/az/README.md <<-EOF +--- +id: az +name: Likvid Bank Azure +azure: + aadTenantId: 1234-1234 + subscriptionId: 1234-1234 +cli: + az: {} +--- +# Likvid Bank Azure +Test test +EOF + +echo "Testing kit features" +collie kit import foundation/docs +collie kit import azure/bootstrap +collie kit apply azure/bootstrap --foundation f --platform az + +collie kit new azure/dummy dummy +cat > ./kit/azure/dummy/variables.tf <<-EOF +variable "my_input_var" {} +EOF + +collie kit apply azure/dummy --foundation f --platform az + +collie foundation deploy --auto-approve + +echo "Testing compliance features" +collie compliance import cfmm + +echo "Testing outputs" +collie foundation tree +collie kit tree +collie compliance tree + +echo "Testing documentation" +collie foundation docs --build