Add tests for kine

Signed-off-by: Vitor Savian <[email protected]>
dereknola · Nov 6, 2024 · 340dc70 · 340dc70
1 parent 541ee59
commit 340dc70
Show file tree

Hide file tree

Showing 5 changed files with 426 additions and 17 deletions.
diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml
@@ -109,13 +109,6 @@ jobs:
     - name: On Failure, Dump Server Logs
       if: ${{ failure() }}
       run: cat ./tests/integration/${{ matrix.itest }}/r2log.txt
-    - name: On Failure, Launch Debug Session
-      uses: dereknola/[email protected]
-      if: ${{ failure() }}
-      with:
-        ## If no one connects after 5 minutes, shut down server.
-        wait-timeout-minutes: 5
-        limit-access-to-actor: true
 
   e2e:
     name: "E2E Tests"
@@ -125,7 +118,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        etest: [dnscache]
+        etest: [dnscache, kine]
       max-parallel: 3
     steps:
     - name: "Checkout"
@@ -168,11 +161,4 @@ jobs:
     - name: Run ${{ matrix.etest }} Test
       run: | 
         cd tests/e2e/${{ matrix.etest }}
-        go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
-    - name: On Failure, Launch Debug Session
-      uses: dereknola/[email protected]
-      if: ${{ failure() }}
-      with:
-        ## If no one connects after 5 minutes, shut down server.
-        wait-timeout-minutes: 5
-        limit-access-to-actor: true
+        go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
diff --git a/tests/e2e/kine/Vagrantfile b/tests/e2e/kine/Vagrantfile
@@ -0,0 +1,157 @@
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
+NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
+  ["server-0", "server-1", "server-2", "agent-0"])
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
+  ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04'])
+GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
+RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+HARDENED = (ENV['E2E_HARDENED'] || "")
+EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "mysql")
+NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 3072).to_i
+CNI = (ENV['E2E_CNI'] || "canal") # canal, cilium and calico supported
+REGISTRY = (ENV['E2E_REGISTRY'] || "")
+# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks,
+# see https://www.virtualbox.org/manual/ch06.html#network_hostonly
+NETWORK_PREFIX = "10.10.10"
+install_type = ""
+
+def provision(vm, role, role_num, node_num)
+  vm.box = NODE_BOXES[node_num]
+  vm.hostname = "#{role[0]}-#{role_num}"
+  # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
+  node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
+  vm.network "private_network", ip: node_ip, netmask: "255.255.255.0"
+
+  scripts_location = Dir.exist?("./scripts") ? "./scripts" : "../scripts" 
+  vagrant_defaults = File.exist?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
+  load vagrant_defaults
+
+  defaultOSConfigure(vm)
+  db_type = getDBType(role, role_num, vm)
+
+  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
+
+  if !HARDENED.empty? 
+    cisPrep(vm)
+  end
+
+  if role.include?("server") && role_num == 0
+    vm.provision :rke2, run: 'once' do |rke2|
+      rke2.env = %W[INSTALL_RKE2_TYPE=server #{install_type}]
+      rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      rke2.config = <<~YAML
+        write-kubeconfig-mode: '0644'
+        node-external-ip: #{NETWORK_PREFIX}.100
+        node-ip: #{NETWORK_PREFIX}.100
+        token: vagrant-rke2
+        cni: #{CNI}
+        #{db_type}
+      YAML
+    end
+  elsif role.include?("server") && role_num != 0
+    vm.provision :rke2, run: 'once' do |rke2|
+      rke2.env = %W[INSTALL_RKE2_TYPE=server #{install_type}]
+      rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      rke2.config = <<~YAML
+        write-kubeconfig-mode: '0644'
+        node-external-ip: #{node_ip}
+        node-ip: #{node_ip}
+        server: https://#{NETWORK_PREFIX}.100:9345
+        token: vagrant-rke2
+        cni: #{CNI}
+        #{db_type}
+      YAML
+    end
+  end
+
+  if role.include?("agent")
+    vm.provision :rke2, run: 'once' do |rke2|
+      rke2.env = %W[INSTALL_RKE2_TYPE=agent #{install_type}]
+      rke2.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      rke2.install_path = false
+      rke2.config = <<~YAML
+        write-kubeconfig-mode: '0644'
+        node-external-ip: #{node_ip}
+        node-ip: #{node_ip}
+        server: https://#{NETWORK_PREFIX}.100:9345
+        token: vagrant-rke2
+      YAML
+    end
+  end
+end
+
+def getDBType(role, role_num, vm)
+  if EXTERNAL_DB == "mariadb"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "shell", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MARIADB_ROOT_PASSWORD=e2e mariadb:11"
+      vm.provision "shell", inline: "echo \"Wait for mariaDB to startup\"; sleep 10"
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/rke2'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/rke2'"
+    end
+
+  elsif EXTERNAL_DB == "mysql"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "shell", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7"
+      vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10"
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/rke2'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/rke2'"
+    end
+
+  elsif EXTERNAL_DB == "postgres"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "shell", type: "shell", inline: "docker run -d -p 5432:5432 --name postgres -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
+      vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10"
+      return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/rke2?sslmode=disable'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/rke2?sslmode=disable'"
+    end
+
+  elsif EXTERNAL_DB == "sqlite"
+    if role.include?("server") && role_num == 0
+      return "--disable-etcd: true"
+    end
+  elsif ( EXTERNAL_DB == "none" )
+    if role.include?("server") && role_num == 0
+      # Will use etcd
+    end
+  else
+    puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
+    abort
+  end
+  return ""
+end
+
+
+Vagrant.configure("2") do |config|
+  config.vagrant.plugins = ["vagrant-rke2", "vagrant-reload"]
+  # Default provider is libvirt, virtualbox is only provided as a backup
+  config.vm.provider "libvirt" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+  config.vm.provider "virtualbox" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+
+  if NODE_ROLES.kind_of?(String)
+    NODE_ROLES = NODE_ROLES.split(" ", -1)
+  end
+  if NODE_BOXES.kind_of?(String)
+    NODE_BOXES = NODE_BOXES.split(" ", -1)
+  end
+
+  NODE_ROLES.each_with_index do |name, i|
+    config.vm.define name do |node|
+      roles = name.split("-", -1)
+      role_num = roles.pop.to_i
+      provision(node.vm, roles, role_num, i)
+    end
+  end
+end