diff --git a/.github/actions/vagrant-setup/action.yaml b/.github/actions/vagrant-setup/action.yaml new file mode 100644 index 0000000000..37f268809e --- /dev/null +++ b/.github/actions/vagrant-setup/action.yaml @@ -0,0 +1,33 @@ +name: 'Setup Vagrant and Libvirt' +description: 'A composite action that installs latest versions of vagrant and libvirt for use on ubuntu based runners' +runs: + using: 'composite' + steps: + - name: Add vagrant to apt-get sources + shell: bash + run: | + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list + - name: Install vagrant and libvirt + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant + sudo systemctl enable --now libvirtd + - name: Build vagrant dependencies + shell: bash + run: | + sudo apt-get build-dep -y vagrant ruby-libvirt + sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev + # This is a workaround for the libvirt group not being available in the current shell + # https://github.com/actions/runner-images/issues/7670#issuecomment-1900711711 + - name: Make the libvirt socket rw accessible to everyone + shell: bash + run: | + sudo chmod a+rw /var/run/libvirt/libvirt-sock + + + - name: Install vagrant-libvirt plugin + shell: bash + run: vagrant plugin install vagrant-libvirt \ No newline at end of file diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index fddde276d5..b438e6f4c5 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -33,9 +33,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Find Go Version for Build + id: go-finder + run: | + GOOS=linux GOARCH=amd64 . ./scripts/version.sh + set +x + VERSION_GOLANG=$(echo $VERSION_GOLANG | sed 's/go//') + echo "VERSION_GOLANG=${VERSION_GOLANG}" >> "$GITHUB_OUTPUT" - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: ${{ steps.go-finder.outputs.VERSION_GOLANG }} cache-dependency-path: | **/go.sum **/go.mod diff --git a/.github/workflows/nightly-install.yaml b/.github/workflows/nightly-install.yaml index b6ca84272a..01bbe9ad2c 100644 --- a/.github/workflows/nightly-install.yaml +++ b/.github/workflows/nightly-install.yaml @@ -6,17 +6,16 @@ on: jobs: test: name: "Smoke Test" - # nested virtualization is only available on macOS hosts - runs-on: macos-12 + runs-on: ubuntu-latest timeout-minutes: 40 strategy: fail-fast: false matrix: channel: [stable] - vm: [centos-7, rocky-8, opensuse-microos, opensuse-leap, ubuntu-focal, windows-2019, windows-2022] + vm: [centos-7, rocky-8, opensuse-leap, ubuntu-2004, windows-2019, windows-2022] include: - {channel: latest, vm: rocky-8} - - {channel: latest, vm: ubuntu-focal} + - {channel: latest, vm: ubuntu-2004} max-parallel: 2 defaults: run: @@ -27,22 +26,25 @@ jobs: - name: "Checkout" uses: actions/checkout@v4 with: {fetch-depth: 1} + # Don't cache Windows VMs, they are 5GB each, which would eat our entire 10GB cache - name: "Vagrant Cache" - uses: actions/cache@v3 + if: ${{ !contains(matrix.vm, 'windows') }} + uses: actions/cache@v4 with: path: | - ~/.vagrant.d/boxes - ~/.vagrant.d/gems - key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }} + ~/.vagrant.d/boxes + key: vagrant-box-${{ matrix.vm }} id: vagrant-cache continue-on-error: true + - name: Set up vagrant and libvirt + uses: ./.github/actions/vagrant-setup - name: "Vagrant Plugin(s)" run: vagrant plugin install vagrant-reload vagrant-rke2 - name: "Vagrant Up ⏩ Install RKE2" run: vagrant up - name: "⏳ Node" if: ${{ !contains(matrix.vm, 'windows') }} - run: vagrant provision --provision-with=rke2-wait-for-node + run: vagrant provision --provision-with=rke2-wait-for-cp - name: "⏳ Canal" if: ${{ !contains(matrix.vm, 'windows') }} run: vagrant provision --provision-with=rke2-wait-for-canal diff --git a/tests/install/centos-7/Vagrantfile b/tests/install/centos-7/Vagrantfile index eaa75a465f..a31b78f4cc 100644 --- a/tests/install/centos-7/Vagrantfile +++ b/tests/install/centos-7/Vagrantfile @@ -4,7 +4,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vm.box = "dweomer/centos-7.9-amd64" + config.vm.box = "generic/centos7" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false %w[libvirt virtualbox vmware_desktop].each do |p| @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config| end end + # Load in helper functions + load "../install_util.rb" + external_env = "" ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} @@ -28,74 +31,15 @@ Vagrant.configure("2") do |config| YAML rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 500 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 300 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + checkRKE2Processes(test.vm) end config.vm.provision "install-packages", type: "shell", run: "once" do |sh| diff --git a/tests/install/install_util.rb b/tests/install/install_util.rb new file mode 100644 index 0000000000..114a71bd65 --- /dev/null +++ b/tests/install/install_util.rb @@ -0,0 +1,90 @@ +def waitForControlPlane(vm, box) + hostname = box.include?("opensuse") ? "$(hostnamectl --static)" : "$(hostname)" + vm.provision "rke2-wait-for-cp", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + echo 'Waiting for node (and static pods) to be ready ...' + time { + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-#{hostname} 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-#{hostname} 2>/dev/null); do sleep 5; done' + } + kubectl get node,all -A -o wide + SHELL + end +end + + +def waitForCanal(vm) + vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def waitForCoreDNS(vm) + vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def waitForIngressNginx(vm) + vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def waitForMetricsServer(vm) + vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time { + timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' + } + SHELL + end +end + +def checkRKE2Processes(vm) + vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep + SHELL + end +end + +def kubectlStatus(vm) + vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + kubectl get node,all -A -o wide + SHELL + end +end \ No newline at end of file diff --git a/tests/install/opensuse-leap/Vagrantfile b/tests/install/opensuse-leap/Vagrantfile index ed63d6074c..8b43db4638 100644 --- a/tests/install/opensuse-leap/Vagrantfile +++ b/tests/install/opensuse-leap/Vagrantfile @@ -4,7 +4,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vm.box = 'opensuse/Leap-15.4.x86_64' + config.vm.box = 'opensuse/Leap-15.5.x86_64' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false %w[libvirt virtualbox vmware_desktop].each do |p| @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config| end end + # Load in helper functions + load "../install_util.rb" + external_env = "" ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} @@ -28,74 +31,15 @@ Vagrant.configure("2") do |config| YAML rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + checkRKE2Processes(test.vm) end config.vm.provision "install-packages", type: "shell", run: "once" do |sh| diff --git a/tests/install/opensuse-microos/README.md b/tests/install/opensuse-microos/README.md deleted file mode 100644 index 500f0af4c3..0000000000 --- a/tests/install/opensuse-microos/README.md +++ /dev/null @@ -1,32 +0,0 @@ -RKE2 Install on MicroOS ---- - -Asserting correctness of the RKE2 installer script using [openSUSE MicroOS](https://microos.opensuse.org/) -as a stand-in for [SUSE Linux Enterprise Micro](https://www.suse.com/products/micro/). - -### Testing With Vagrant - -The [Vagrant box](https://app.vagrantup.com/dweomer/boxes/microos.amd64) used for this test supports these providers: -- `libvirt` -- `virtualbox` (the default for most installations, including `macos-12` github actions runners) -- `vmware_desktop` - -To spin up a VM to test a locally modified `install.sh`: -```shell -# make sure the vagrant-reload plugin is installed, one-time only -vagrant plugin install vagrant-reload -``` -```shell -vagrant up -``` - -See also: -- [developer-docs/testing.md](../../../developer-docs/testing.md#environment-variables) - -### Vagrant Reload Plugin - -The MicroOS guest leverages [transactional updates](https://documentation.suse.com/sles/15-SP1/html/SLES-all/cha-transactional-updates.html) -for most persistent mutations of the installation (typically involving the `/usr` partition) which requires a reboot to -take effect. The `vagrant-reload` provisioner plugin is used for this because the implementation of the [`reboot` option](https://www.vagrantup.com/docs/provisioning/shell#reboot) -of the built-in [Vagrant Shell Provisioner](https://www.vagrantup.com/docs/provisioning/shell) is unreliable -(especially if used more than once per provisioning run). diff --git a/tests/install/opensuse-microos/Vagrantfile b/tests/install/opensuse-microos/Vagrantfile deleted file mode 100644 index d02e504719..0000000000 --- a/tests/install/opensuse-microos/Vagrantfile +++ /dev/null @@ -1,140 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' - -Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-reload' => {}, - } - config.vm.box = "dweomer/microos.amd64" - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false - %w[libvirt virtualbox vmware_desktop].each do |p| - config.vm.provider p do |v, o| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 3072 - end - end - - external_env = "" - ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} - - config.vm.define "install-microos", primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision 'rke2-upload-installer', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' - test.vm.provision"rke2-install", type: 'rke2', run: "once" do |rke2| - rke2.installer_url = 'file:///home/vagrant/install.sh' - rke2.env = %W[ #{external_env} INSTALL_RKE2_TYPE=server] - rke2.config = <<~YAML - token: 'vagrant' - YAML - rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end - end - - config.vm.provision "install-packages", type: "shell", run: "once" do |sh| - sh.upload_path = "/tmp/vagrant-install-packages" - sh.env = { - 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - source /etc/profile.d/rke2.sh - set -eux -o pipefail - transactional-update --no-selfupdate -d pkg install -y --allow-unsigned-rpm \ - curl \ - iptables \ - less \ - lsof \ - socat \ - ${INSTALL_PACKAGES} - SHELL - end - config.vm.provision "install-packages-reload", type: "reload", run: "once" - config.vm.provision "selinux-status", type: "shell", run: "once", inline: "sestatus -v" - config.vm.provision "rke2-profile-env", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - cat <<-EOF > /etc/profile.d/rke2.sh -export KUBECONFIG=/etc/rancher/rke2/rke2.yaml PATH=/usr/local/bin:$PATH:/var/lib/rancher/rke2/bin -EOF - SHELL - end - -end diff --git a/tests/install/rocky-8/Vagrantfile b/tests/install/rocky-8/Vagrantfile index d7938edfad..02ec340aa6 100644 --- a/tests/install/rocky-8/Vagrantfile +++ b/tests/install/rocky-8/Vagrantfile @@ -14,6 +14,9 @@ Vagrant.configure("2") do |config| end end + # Load in helper functions + load "../install_util.rb" + external_env = "" ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} @@ -29,74 +32,15 @@ Vagrant.configure("2") do |config| YAML rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|spc_t|unconfined_t' | grep -v grep - SHELL - end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + checkRKE2Processes(test.vm) end config.vm.provision "install-packages", type: "shell", run: "once" do |sh| diff --git a/tests/install/ubuntu-focal/README.md b/tests/install/ubuntu-2004/README.md similarity index 93% rename from tests/install/ubuntu-focal/README.md rename to tests/install/ubuntu-2004/README.md index 2b94b7f9d7..9b6719ad65 100644 --- a/tests/install/ubuntu-focal/README.md +++ b/tests/install/ubuntu-2004/README.md @@ -1,4 +1,4 @@ -RKE2 Install on Ubuntu Focal Fossa +RKE2 Install on Ubuntu 20.04 Focal Fossa --- Asserting correctness of the RKE2 installer script on [Ubuntu 20.04](https://releases.ubuntu.com/20.04/). diff --git a/tests/install/ubuntu-2004/Vagrantfile b/tests/install/ubuntu-2004/Vagrantfile new file mode 100644 index 0000000000..92ea543007 --- /dev/null +++ b/tests/install/ubuntu-2004/Vagrantfile @@ -0,0 +1,80 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' + +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2004" + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false + %w[libvirt virtualbox vmware_desktop].each do |p| + config.vm.provider p do |v, o| + v.cpus = ENV['TEST_VM_CPUS'] || 2 + v.memory = ENV['TEST_VM_MEMORY'] || 3072 + end + end + + # Load in helper functions + load "../install_util.rb" + + external_env = "" + ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} + + config.vm.define "install-ubuntu-2004", primary: true do |test| + test.vm.hostname = 'smoke' + test.vm.provision 'rke2-upload-installer', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' + test.vm.provision"rke2-install", type: 'rke2', run: "once" do |rke2| + rke2.installer_url = 'file:///home/vagrant/install.sh' + rke2.env = %W[ #{external_env} INSTALL_RKE2_TYPE=server] + rke2.config = <<~YAML + token: 'vagrant' + YAML + rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + + waitForControlPlane(test.vm, config.vm.box.to_s) + waitForCanal(test.vm) + waitForCoreDNS(test.vm) + waitForIngressNginx(test.vm) + waitForMetricsServer(test.vm) + + kubectlStatus(test.vm) + test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'etcd|kube|rke2|container|confined' | grep -v grep + SHELL + end + end + + config.vm.provision "install-packages", type: "shell", run: "once" do |sh| + sh.upload_path = "/tmp/vagrant-install-packages" + sh.env = { + 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + apt-get -y update + apt-get -y install \ + curl \ + iptables \ + less \ + lsof \ + netcat \ + socat \ + ${INSTALL_PACKAGES} + SHELL + end + + config.vm.provision "rke2-profile-env", type: "shell", run: "once" do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + cat <<-EOF > /etc/profile.d/rke2.sh +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml PATH=/usr/local/bin:$PATH:/var/lib/rancher/rke2/bin +EOF + SHELL + end + +end diff --git a/tests/install/ubuntu-focal/Vagrantfile b/tests/install/ubuntu-focal/Vagrantfile deleted file mode 100644 index 63ad652759..0000000000 --- a/tests/install/ubuntu-focal/Vagrantfile +++ /dev/null @@ -1,130 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' - -Vagrant.configure("2") do |config| - config.vm.box = "generic/ubuntu2004" - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '.', '/vagrant', type: 'rsync', disabled: false - %w[libvirt virtualbox vmware_desktop].each do |p| - config.vm.provider p do |v, o| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 3072 - end - end - - external_env = "" - ENV.select{|k,v| k.start_with?('RKE2_') || k.start_with?('INSTALL_RKE2_')}.each {|key,value| external_env << "#{key.to_s}=#{value.to_s}"} - - config.vm.define "install-ubuntu-focal", primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision 'rke2-upload-installer', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' - test.vm.provision"rke2-install", type: 'rke2', run: "once" do |rke2| - rke2.installer_url = 'file:///home/vagrant/install.sh' - rke2.env = %W[ #{external_env} INSTALL_RKE2_TYPE=server] - rke2.config = <<~YAML - token: 'vagrant' - YAML - rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "rke2-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node (and static pods) to be ready ...' - time { - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/etcd-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-apiserver-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-scheduler-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-proxy-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/kube-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl wait --for condition=ready -n kube-system pod/cloud-controller-manager-$(hostname) 2>/dev/null); do sleep 5; done' - } - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-wait-for-canal", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-canal 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns 2>/dev/null); do sleep 5; done' - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-coredns-rke2-coredns-autoscaler 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-ingress-nginx", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s daemonset/rke2-ingress-nginx-controller 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time { - timeout 240 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/rke2-metrics-server 2>/dev/null); do sleep 5; done' - } - SHELL - end - test.vm.provision "rke2-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "rke2-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'etcd|kube|rke2|container|confined' | grep -v grep - SHELL - end - end - - config.vm.provision "install-packages", type: "shell", run: "once" do |sh| - sh.upload_path = "/tmp/vagrant-install-packages" - sh.env = { - 'INSTALL_PACKAGES': ENV['INSTALL_PACKAGES'], - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - apt-get -y update - apt-get -y install \ - curl \ - iptables \ - less \ - lsof \ - netcat \ - socat \ - ${INSTALL_PACKAGES} - SHELL - end - - config.vm.provision "rke2-profile-env", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - cat <<-EOF > /etc/profile.d/rke2.sh -export KUBECONFIG=/etc/rancher/rke2/rke2.yaml PATH=/usr/local/bin:$PATH:/var/lib/rancher/rke2/bin -EOF - SHELL - end - -end diff --git a/tests/install/windows-2019/Vagrantfile b/tests/install/windows-2019/Vagrantfile index 8991aa0890..68a7d7e04e 100644 --- a/tests/install/windows-2019/Vagrantfile +++ b/tests/install/windows-2019/Vagrantfile @@ -5,7 +5,7 @@ ENV['TEST_INSTALL_PS1'] ||= '../../../install.ps1' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-reload"] - config.vm.box = "gusztavvargadr/windows-server-2019-standard" + config.vm.box = "jborean93/WindowsServer2019" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true %w[libvirt virtualbox hyperv].each do |p|