Skip to content

Commit

Permalink
Rework and fix nightly install tests (#5692)
Browse files Browse the repository at this point in the history
* Remove microos install test
* Rename ubuntu install test
* Consolidate install testing checks
* Move to linux GitHub Action runners
* Bump leap to 15.5
* Bump Windows 2019 to libvirt image
Signed-off-by: Derek Nola <[email protected]>

* Fix go version for integration tests

Signed-off-by: Derek Nola <[email protected]>

---------

Signed-off-by: Derek Nola <[email protected]>
  • Loading branch information
dereknola committed Apr 17, 2024
1 parent e7a6404 commit f562675
Show file tree
Hide file tree
Showing 13 changed files with 262 additions and 520 deletions.
33 changes: 33 additions & 0 deletions .github/actions/vagrant-setup/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: 'Setup Vagrant and Libvirt'
description: 'A composite action that installs latest versions of vagrant and libvirt for use on ubuntu based runners'
runs:
using: 'composite'
steps:
- name: Add vagrant to apt-get sources
shell: bash
run: |
curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list
- name: Install vagrant and libvirt
shell: bash
run: |
sudo apt-get update
sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant
sudo systemctl enable --now libvirtd
- name: Build vagrant dependencies
shell: bash
run: |
sudo apt-get build-dep -y vagrant ruby-libvirt
sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
# This is a workaround for the libvirt group not being available in the current shell
# https://github.com/actions/runner-images/issues/7670#issuecomment-1900711711
- name: Make the libvirt socket rw accessible to everyone
shell: bash
run: |
sudo chmod a+rw /var/run/libvirt/libvirt-sock

- name: Install vagrant-libvirt plugin
shell: bash
run: vagrant plugin install vagrant-libvirt
9 changes: 8 additions & 1 deletion .github/workflows/integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,16 @@ jobs:
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Find Go Version for Build
id: go-finder
run: |
GOOS=linux GOARCH=amd64 . ./scripts/version.sh
set +x
VERSION_GOLANG=$(echo $VERSION_GOLANG | sed 's/go//')
echo "VERSION_GOLANG=${VERSION_GOLANG}" >> "$GITHUB_OUTPUT"
- uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
go-version: ${{ steps.go-finder.outputs.VERSION_GOLANG }}
cache-dependency-path: |
**/go.sum
**/go.mod
Expand Down
20 changes: 11 additions & 9 deletions .github/workflows/nightly-install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,16 @@ on:
jobs:
test:
name: "Smoke Test"
# nested virtualization is only available on macOS hosts
runs-on: macos-12
runs-on: ubuntu-latest
timeout-minutes: 40
strategy:
fail-fast: false
matrix:
channel: [stable]
vm: [centos-7, rocky-8, opensuse-microos, opensuse-leap, ubuntu-focal, windows-2019, windows-2022]
vm: [centos-7, rocky-8, opensuse-leap, ubuntu-2004, windows-2019, windows-2022]
include:
- {channel: latest, vm: rocky-8}
- {channel: latest, vm: ubuntu-focal}
- {channel: latest, vm: ubuntu-2004}
max-parallel: 2
defaults:
run:
Expand All @@ -27,22 +26,25 @@ jobs:
- name: "Checkout"
uses: actions/checkout@v4
with: {fetch-depth: 1}
# Don't cache Windows VMs, they are 5GB each, which would eat our entire 10GB cache
- name: "Vagrant Cache"
uses: actions/cache@v3
if: ${{ !contains(matrix.vm, 'windows') }}
uses: actions/cache@v4
with:
path: |
~/.vagrant.d/boxes
~/.vagrant.d/gems
key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }}
~/.vagrant.d/boxes
key: vagrant-box-${{ matrix.vm }}
id: vagrant-cache
continue-on-error: true
- name: Set up vagrant and libvirt
uses: ./.github/actions/vagrant-setup
- name: "Vagrant Plugin(s)"
run: vagrant plugin install vagrant-reload vagrant-rke2
- name: "Vagrant Up ⏩ Install RKE2"
run: vagrant up
- name: "⏳ Node"
if: ${{ !contains(matrix.vm, 'windows') }}
run: vagrant provision --provision-with=rke2-wait-for-node
run: vagrant provision --provision-with=rke2-wait-for-cp
- name: "⏳ Canal"
if: ${{ !contains(matrix.vm, 'windows') }}
run: vagrant provision --provision-with=rke2-wait-for-canal
Expand Down
82 changes: 13 additions & 69 deletions tests/install/centos-7/Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'

Vagrant.configure("2") do |config|
config.vm.box = "dweomer/centos-7.9-amd64"
config.vm.box = "generic/centos7"
config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds
config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false
%w[libvirt virtualbox vmware_desktop].each do |p|
Expand All @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config|
end
end

# Load in helper functions
load "../install_util.rb"

external_env = ""
ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"}

Expand All @@ -28,74 +31,15 @@ Vagrant.configure("2") do |config|
YAML
rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
end
test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
echo 'Waiting for node (and static pods) to be ready ...'
time {
timeout 500 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done'
timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done'
timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done'
timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done'
timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done'
timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done'
timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done'
}
kubectl get node,all -A -o wide
SHELL
end
test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done'
}
SHELL
end
test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done'
timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done'
}
SHELL
end
test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done'
}
SHELL
end
test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done'
}
SHELL
end
test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eux -o pipefail
kubectl get node,all -A -o wide
SHELL
end
test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eux -o pipefail
ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep
SHELL
end

waitForControlPlane(test.vm, config.vm.box.to_s)
waitForCanal(test.vm)
waitForCoreDNS(test.vm)
waitForIngressNginx(test.vm)
waitForMetricsServer(test.vm)

kubectlStatus(test.vm)
checkRKE2Processes(test.vm)
end

config.vm.provision "install-packages", type: "shell", run: "once" do |sh|
Expand Down
90 changes: 90 additions & 0 deletions tests/install/install_util.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
def waitForControlPlane(vm, box)
hostname = box.include?("opensuse") ? "$(hostnamectl --static)" : "$(hostname)"
vm.provision "rke2-wait-for-cp", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
echo 'Waiting for node (and static pods) to be ready ...'
time {
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/#{hostname} 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-#{hostname} 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-#{hostname} 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-#{hostname} 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-#{hostname} 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-#{hostname} 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-#{hostname} 2>/dev/null); do sleep 5; done'
}
kubectl get node,all -A -o wide
SHELL
end
end


def waitForCanal(vm)
vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done'
}
SHELL
end
end

def waitForCoreDNS(vm)
vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done'
timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done'
}
SHELL
end
end

def waitForIngressNginx(vm)
vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done'
}
SHELL
end
end

def waitForMetricsServer(vm)
vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eu -o pipefail
time {
timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done'
}
SHELL
end
end

def checkRKE2Processes(vm)
vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eux -o pipefail
ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep
SHELL
end
end

def kubectlStatus(vm)
vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eux -o pipefail
kubectl get node,all -A -o wide
SHELL
end
end
Loading

0 comments on commit f562675

Please sign in to comment.