test: add e2e tests for windows and linux

- add COLLIE_VERSION env var support to install scripts
- fix a typo in collie compliance tree warning

.gitattributes

@@ -0,0 +1,6 @@
+# Force LF file endings for all text files
+* text eol=lf
+
+# Denote all files that are truly binary and should not be modified.
+*.png binary
+*.jpg binary

.github/workflows/e2e.yml

@@ -0,0 +1,68 @@
+name: e2e
+on:
+  # temporary, just for testing the workflow
+  push:
+    branches: [ feature/e2e-tests ]
+  # run after the "release" workflow https://stackoverflow.com/a/64733705/125407
+  workflow_run:
+    workflows: ["release"]
+    branches: [main]
+    types: 
+      - completed
+
+# TODO: replace "feature/e2e-tests" with "main" before merging to main branch!
+
+jobs:
+  # this test is super basic and just checks whether collie blows up
+  e2e-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: install dependencies
+        uses: crazy-max/ghaction-chocolatey@v3
+        # note: all other dependencies are installed in the image already
+        with:
+          args: install --force terraform terragrunt terraform-docs
+      - name: install collie
+        shell: pwsh
+        env: 
+          COLLIE_VERSION: ${{ github.event.release.tag_name }}
+        run: |
+          irm https://raw.githubusercontent.com/meshcloud/collie-cli/feature/e2e-tests/install.ps1 | iex
+          
+          # github doesn't allow adding to PATH any other way, see https://stackoverflow.com/a/71579543/125407
+          Add-Content $env:GITHUB_PATH "C:\Users\runneradmin\collie-cli"
+      - name: collie info
+        shell: pwsh
+        run: |
+          collie info
+      - name: test
+        shell: pwsh
+        run: .\test\e2e.ps1
+  e2e-linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: install dependencies 
+        # note: we forego default.nix and to be as close to a realistic user experience as possible
+        # all the other dependencies are already present on the runner images, so we use those
+        # and just install the missing ones via nix
+        uses: cachix/install-nix-action@v18
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          extra_nix_config: |
+            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+      - uses: rrbutani/use-nix-shell-action@v1
+        with:
+          packages: terragrunt terraform-docs
+      - name: install collie
+        env: 
+          COLLIE_VERSION: ${{ github.event.release.tag_name }}
+        run: |
+          curl -sf -L https://raw.githubusercontent.com/meshcloud/collie-cli/main/install.sh | sudo bash
+      - name: collie info
+        run: |
+          collie info
+      - name: test
+        run: |
+          ./test/e2e.sh

.github/workflows/release.yml

@@ -2,7 +2,7 @@ name: release
 
 on:
   release:
-    types: [published]
+    types: [prereleased]
 
 jobs:
   create-release:

CONTRIBUTING.md

@@ -41,16 +41,20 @@ The versions have this format `vMAJOR.MINOR.PATCH`, e.g. `v0.1.0`.
 
 The release is triggered by
 [creating a new release via GitHub's UI](https://github.com/meshcloud/collie-cli/releases/new)
-as this gives us a good workflow to review and edit release notes before
-publishing.
+**as a prerelease**. This gives us a good workflow to review and edit release
+notes before publishing as well as running end to end tests before publishing
+the release.
 
 The full workflow from forth to back is as follows:
 
 - decide on the version number you want to use for the release choose a new tag
   "vMAJOR.MINOR.PATCH"
-- click "Generate release notes"
+- click "Generate release notes" and tick *
 - the release binaries are created automatically by the
   [release workflow](.github/workflows/releases.yml)
+- the e2e tests are run after the release workflow completes
+  [e2e workflow](.github/workflows/e2e.yml)
+- manually inspect the e2e test outcomes and publish the release
 
 > Note: The install script also always works with thse latest version, so you're
 > done. Nice! 🎉

install.ps1

@@ -1,15 +1,21 @@
-Clear-Host
 Write-Host "`nStarting Installation of Collie ..." -ForegroundColor Green
 
 $ProgressPreference = "SilentlyContinue" # Don't remove this! Otherwise it will slow down the Download-Process under PS 5
 
 # Check the latest release to get the exact download URL
 try {
-    $query = Invoke-RestMethod -Uri "https://api.github.com/repos/meshcloud/collie-cli/releases/latest"
+    $uri = "https://api.github.com/repos/meshcloud/collie-cli/releases/latest"
+    if (![string]::IsNullOrEmpty($env:COLLIE_VERSION )) {
+        $uri = "https://api.github.com/repos/meshcloud/collie-cli/releases/tags/$($env:COLLIE_VERSION)"
+    }
+
+    Write-Host "Searching for collie release at $uri"
+    $query = Invoke-RestMethod -Uri $uri
     $downloadUrl = $query.assets | Where-Object { $_.browser_download_url -match "windows-msvc.exe" }
 }
 catch {
-    throw "[ERROR] while downloading installation file. Check your Internet Connection: $(_.Exception.Message)!"
+    Write-Host "[ERROR] while locating collie release!"
+    throw
 }
 
 $installationPath = "${HOME}\collie-cli"
@@ -23,7 +29,7 @@ if ( !(Test-Path -Path $installationExec) ) {
 }
 elseif ( Test-Path -Path $installationExec) {
     $replace = $(Write-Host "Collie already exists. Do you want to replace the existing Installation (y/n)? " -NoNewLine -ForegroundColor Green; Read-Host) 
-    if ($replace -like "y") {
+    if ($replace -eq "y") {
         Remove-Item $installationExec -Force
     }
     else { Exit } 
@@ -36,7 +42,8 @@ try {
     if ($?) { Write-Host "[OK] Download-Size '$([math]::Round($download.RawContentLength/1MB)),2 MB'`n" -ForegroundColor Green }
 }
 catch {
-    throw "[ERROR] while downloading Installation File: $($($_.Exception).Message)!"
+    Write-Host "[ERROR] while downloading installation file!"
+    throw
 }
 
 # Check if Collie is already in PATH
@@ -47,7 +54,7 @@ if (!(";$Path;".ToLower() -like "*;$BinDir;*".ToLower())) {
 
     # Ask user whether to add Collie to PATH
     $userenv = $(Write-Host "Could not find collie CLI on PATH. Do you want to add collie CLI to PATH? (y/n)" -NoNewLine -ForegroundColor Green; Read-Host)
-    if ($userenv -like "y") {
+    if ($userenv -eq "y") {
         Write-Host "Adding $BinDir to PATH" -ForegroundColor Green
         [Environment]::SetEnvironmentVariable('Path', "$Path;$BinDir", $User)
         $Env:Path += ";$BinDir"

install.sh

@@ -21,15 +21,22 @@ for i in $REQ_COMMANDS
 do
   command -v "$i" >/dev/null && continue || { echo "$i command not found. You need to install if before running this script"; exit 1; }
 done
+
+COLLIE_VERSION=${COLLIE_VERSION:-}
+release_url="https://api.github.com/repos/meshcloud/collie-cli/releases/latest"
+if [ ! -z "${COLLIE_VERSION}" ]; then
+  release_url="https://api.github.com/repos/meshcloud/collie-cli/releases/tags/${COLLIE_VERSION}"
+fi
+
 # Download artifacts based on CPU architecture and OS type
 case "$(uname -s)" in
    Darwin)
-      url=$(curl -s https://api.github.com/repos/meshcloud/collie-cli/releases/latest | grep "browser_download_url.*$arch.*apple" | cut -d : -f 2,3 | tr -d \" | tr -d \ )
+      artifact_url=$(curl -s "$release_url" | grep "browser_download_url.*$arch.*apple" | cut -d : -f 2,3 | tr -d \" | tr -d \ )
       name="collie-$arch-apple-darwin"    
      ;;
    Linux)
     echo "detected Linux"
-     url=$(curl -s https://api.github.com/repos/meshcloud/collie-cli/releases/latest | grep "browser_download_url.*linux" | cut -d : -f 2,3 | tr -d \" | tr -d \ )
+     artifact_url=$(curl -s "$release_url" | grep "browser_download_url.*linux" | cut -d : -f 2,3 | tr -d \" | tr -d \ )
      name="collie-x86_64-unknown-linux-gnu"
      ;;
    CYGWIN*|MINGW32*|MSYS*|MINGW*)
@@ -42,8 +49,8 @@ case "$(uname -s)" in
      ;;
 esac
 
-echo "Downloading the artifact... (${url})"
-curl "${url}" -L -o collie.tar.gz --silent
+echo "Downloading the artifact... (${artifact_url})"
+curl "${artifact_url}" -L -o collie.tar.gz --silent
 # untar and movin the artifact
 echo "Unpacking the artifact and copying the binary..."
 tar -zxf collie.tar.gz -C /usr/local/bin

src/commands/compliance/tree.command.ts

@@ -50,7 +50,7 @@ export function registerTreeCmd(program: TopLevelCommand) {
       );
 
       if (!hasAnyApplicableStatements) {
-        logger.warn("no compliance control statements found in an< kit module");
+        logger.warn("no compliance control statements found in any kit module");
         logger.tip(
           `Add a compliance section to your kit module frontmatter like this\n` +
             statementExample,

test/e2e.ps1

@@ -0,0 +1,61 @@
+$tmpdir=New-TemporaryFile | %{ rm $_; mkdir $_ }
+echo "Running test in $tmpdir - please remember to cleanup"
+
+cd $tmpdir
+
+# don't ask..., if you must - https://stackoverflow.com/questions/5596982/using-powershell-to-write-a-file-in-utf-8-without-the-bom#comment79966549_5596984
+[System.Environment]::CurrentDirectory = (Get-Location).Path
+
+collie init .
+
+echo "Testing foundation features"
+collie foundation new f
+
+# fake a platform config file
+
+# equivalent of mkdir -p
+md foundations/f/platforms/az/ -ea 0
+
+# powershell needs some convicing to write files at UTF8 without BOM... this is the only thing that works
+$platform_md=@"
+---
+id: az
+name: Likvid Bank Azure
+azure:
+    aadTenantId: 1234-1234
+    subscriptionId: 1234-1234
+cli: 
+    az: {}
+---
+# Likvid Bank Azure
+Test test
+"@
+[System.IO.File]::WriteAllLines("foundations/f/platforms/az/README.md", $platform_md)
+
+
+echo "Testing kit features"
+collie kit import foundation/docs
+collie kit import azure/bootstrap
+collie kit apply azure/bootstrap --foundation f --platform az
+
+collie kit new azure/dummy dummy
+
+$variables_tf=@"
+variable "my_input_var" {}
+"@
+[IO.File]::WriteAllLines("kit/azure/dummy/variables.tf", $variables_tf)
+
+collie kit apply azure/dummy --foundation f --platform az
+
+collie foundation deploy --auto-approve
+
+echo "Testing compliance features"
+collie compliance import cfmm
+
+echo "Testing outputs"
+collie foundation tree
+collie kit tree
+collie compliance tree
+
+echo "Testing documentation"
+collie foundation docs --build

test/e2e.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -o errexit   # exit on error
+set -o errtrace  # enables ERR traps so we can run cleanup
+set -o pipefail  # exit on error in a pipe, without this only the status of the last command in a pipe is considered
+set -o nounset   # exit on undefined variables
+
+tmpdir=$(mktemp -d)
+echo "Running test in $tmpdir - please remember to cleanup"
+
+cd "$tmpdir"
+
+collie init .
+
+echo "Testing foundation features"
+collie foundation new f > /dev/null
+
+# fake a platform config file
+mkdir -p ./foundations/f/platforms/az/
+cat > ./foundations/f/platforms/az/README.md <<-EOF
+---
+id: az
+name: Likvid Bank Azure
+azure:
+    aadTenantId: 1234-1234
+    subscriptionId: 1234-1234
+cli:
+    az: {}
+---
+# Likvid Bank Azure
+Test test
+EOF
+
+echo "Testing kit features"
+collie kit import foundation/docs
+collie kit import azure/bootstrap
+collie kit apply azure/bootstrap --foundation f --platform az
+
+collie kit new azure/dummy dummy
+cat > ./kit/azure/dummy/variables.tf <<-EOF
+variable "my_input_var" {}
+EOF
+
+collie kit apply azure/dummy --foundation f --platform az
+
+collie foundation deploy --auto-approve
+
+echo "Testing compliance features"
+collie compliance import cfmm
+
+echo "Testing outputs"
+collie foundation tree
+collie kit tree
+collie compliance tree
+
+echo "Testing documentation"
+collie foundation docs --build