Simplify HAproxy and upgrade provider

README.md

@@ -6,7 +6,7 @@ A simple kubernetes cluster using Fedora Core OS, Proxmox and k0sctl.
 
 ## Dependencies
 
-`Client` refers to the node that will be executing `terraform apply` to create the cluster. The `Raspberry Pi` can be replaced with a VM or a LXC container. The items marked `Optional` are needed only when you want to expose your kubernetes services to the internet via WireGuard.
+`Client` refers to the node that will be executing `terraform apply` to create the cluster.
 
 | Dependency | Location |
 | ------ | ------ |
@@ -15,9 +15,6 @@ A simple kubernetes cluster using Fedora Core OS, Proxmox and k0sctl.
 | [jq](https://stedolan.github.io/jq/) | Client |
 | [Terraform](https://www.terraform.io/) | Client |
 | [k0sctl](https://github.com/k0sproject/k0sctl) | Client |
-| [HAproxy](http://www.haproxy.org/) | Raspberry Pi |
-| [Wireguard](https://www.wireguard.com/) (Optional) | Raspberry Pi & Cloud VPS |
-| [Docker](https://docs.docker.com/) (Optional) | Cloud VPS |
 
 ## One-time Configuration
 
@@ -31,12 +28,6 @@ cd simple-fcos-cluster/scripts
 chmod +x ./versions.sh
 ```
 
-### Create an HA Proxy Server
-
-I've installed `haproxy` on my Raspberry Pi. You can choose to do the same in a LXC container or a VM.
-
-You need to have passwordless SSH access to a user (from the Client node) in this node which has the permissions to modify the file `/etc/haproxy/haproxy.cfg` and permissions to run `sudo systemctl restart haproxy`. An example is covered in this [doc](https://github.com/Naman1997/simple-fcos-cluster/blob/main/docs/HA_Proxy.md).
-
 
 ### Create the terraform.tfvars file
 

main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     proxmox = {
       source  = "bpg/proxmox"
-      version = "0.57.1"
+      version = "0.61.1"
     }
   }
 }
@@ -21,9 +21,10 @@ data "external" "versions" {
 }
 
 locals {
-  fcos_version = data.external.versions.result["fcos_version"]
-  k0s_version  = data.external.versions.result["k0s_version"]
-  iso_url      = "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${local.fcos_version}/x86_64/fedora-coreos-${local.fcos_version}-qemu.x86_64.qcow2.xz"
+  ha_proxy_user = "ubuntu"
+  fcos_version  = data.external.versions.result["fcos_version"]
+  k0s_version   = data.external.versions.result["k0s_version"]
+  iso_url       = "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${local.fcos_version}/x86_64/fedora-coreos-${local.fcos_version}-qemu.x86_64.qcow2.xz"
 }
 
 resource "null_resource" "download_fcos_image" {
@@ -165,12 +166,20 @@ module "worker_domain" {
   target_node    = var.TARGET_NODE
 }
 
+module "proxy" {
+  source         = "./modules/proxy"
+  ha_proxy_user  = local.ha_proxy_user
+  DEFAULT_BRIDGE = var.DEFAULT_BRIDGE
+  TARGET_NODE    = var.TARGET_NODE
+}
+
 
 resource "local_file" "haproxy_config" {
 
   depends_on = [
     module.master_domain.node,
-    module.worker_domain.node
+    module.worker_domain.node,
+    module.proxy.node
   ]
 
   content = templatefile("${path.root}/templates/haproxy.tmpl",
@@ -190,16 +199,16 @@ resource "local_file" "haproxy_config" {
     destination = "/etc/haproxy/haproxy.cfg"
     connection {
       type        = "ssh"
-      host        = var.ha_proxy_server
-      user        = var.ha_proxy_user
+      host        = module.proxy.proxy_ipv4_address
+      user        = local.ha_proxy_user
       private_key = file("~/.ssh/id_rsa")
     }
   }
 
   provisioner "remote-exec" {
     connection {
-      host        = var.ha_proxy_server
-      user        = var.ha_proxy_user
+      host        = module.proxy.proxy_ipv4_address
+      user        = local.ha_proxy_user
       private_key = file("~/.ssh/id_rsa")
     }
 
@@ -225,7 +234,7 @@ resource "local_file" "k0sctl_config" {
       ),
       "user"        = "core",
       "k0s_version" = local.k0s_version,
-      "ha_proxy_server" : var.ha_proxy_server
+      "ha_proxy_server" : module.proxy.proxy_ipv4_address
     }
   )
   filename = "k0sctl.yaml"
@@ -239,7 +248,22 @@ resource "null_resource" "setup_cluster" {
 
   provisioner "local-exec" {
     command = <<-EOT
-      k0sctl apply --config k0sctl.yaml --disable-telemetry
+      MAX_RETRIES=5
+      RETRY_INTERVAL=10
+      for ((i = 1; i <= MAX_RETRIES; i++)); do
+        k0sctl apply --config k0sctl.yaml --disable-telemetry
+        code=$?
+        if [ $code -eq 0 ]; then
+          break
+        fi
+        if [ $i -lt $MAX_RETRIES ]; then
+          echo "Unable to apply config. Retrying in 10 seconds..."
+          sleep $RETRY_INTERVAL
+        else
+          echo "Maximum retries reached. Unable to apply config."
+          exit 1
+        fi
+      done
       mkdir -p ~/.kube
       k0sctl kubeconfig > ~/.kube/config --disable-telemetry
       chmod 600 ~/.kube/config

modules/domain/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     proxmox = {
       source  = "bpg/proxmox"
-      version = "0.57.1"
+      version = "0.61.1"
     }
   }
 }
@@ -27,7 +27,7 @@ resource "proxmox_virtual_environment_vm" "node" {
 
   agent {
     enabled = true
-    timeout = "120s"
+    timeout = "600s"
   }
 
   clone {

modules/proxy/main.tf

@@ -0,0 +1,136 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "0.61.1"
+    }
+  }
+}
+
+data "local_file" "ssh_public_key" {
+  filename = pathexpand("~/.ssh/id_rsa.pub")
+}
+
+resource "proxmox_virtual_environment_file" "cloud_config" {
+  content_type = "snippets"
+  datastore_id = "local"
+  node_name    = var.TARGET_NODE
+
+  source_raw {
+    data = <<-EOF
+    #cloud-config
+    users:
+      - default
+      - name: ${var.ha_proxy_user}
+        groups:
+          - sudo
+        shell: /bin/bash
+        ssh_authorized_keys:
+          - ${trimspace(data.local_file.ssh_public_key.content)}
+        sudo: ALL=(ALL) NOPASSWD:ALL
+    runcmd:
+        - apt update -y && apt dist-upgrade -y
+        - apt install -y qemu-guest-agent haproxy net-tools unattended-upgrades
+        - timedatectl set-timezone America/Toronto
+        - systemctl enable qemu-guest-agent
+        - systemctl enable --now haproxy
+        - systemctl start qemu-guest-agent
+        - chown -R ${var.ha_proxy_user}:${var.ha_proxy_user} /etc/haproxy/
+        - echo "done" > /tmp/cloud-config.done
+    EOF
+
+    file_name = "cloud-config.yaml"
+  }
+}
+
+resource "proxmox_virtual_environment_download_file" "ubuntu_cloud_image" {
+  content_type = "iso"
+  datastore_id = "local"
+  node_name    = var.TARGET_NODE
+
+  url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+}
+
+resource "proxmox_virtual_environment_vm" "node" {
+  name      = "haproxy"
+  node_name = var.TARGET_NODE
+
+  agent {
+    enabled = true
+  }
+
+  cpu {
+    cores = 2
+  }
+
+  memory {
+    dedicated = 2048
+  }
+
+  disk {
+    datastore_id = "local-lvm"
+    file_id      = proxmox_virtual_environment_download_file.ubuntu_cloud_image.id
+    interface    = "virtio0"
+    iothread     = true
+    discard      = "on"
+    size         = 20
+  }
+
+  initialization {
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
+    }
+
+    user_data_file_id = proxmox_virtual_environment_file.cloud_config.id
+  }
+
+  network_device {
+    bridge = var.DEFAULT_BRIDGE
+  }
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      n=0
+      until [ "$n" -ge 10 ]
+      do
+        echo "Attempt number: $n"
+        ssh-keygen -R $ADDRESS
+        if [ $? -eq 0 ]; then
+          echo "Successfully removed $ADDRESS"
+          break
+        fi
+        n=$((n+1)) 
+        sleep $[ ( $RANDOM % 10 )  + 1 ]s
+      done
+    EOT
+    environment = {
+      ADDRESS = element(flatten(self.ipv4_addresses), 1)
+    }
+    when = destroy
+  }
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      n=0
+      until [ "$n" -ge 10 ]
+      do
+        echo "Attempt number: $n"
+        ssh-keyscan -H $ADDRESS >> ~/.ssh/known_hosts
+        ssh -q -o StrictHostKeyChecking=no ${var.ha_proxy_user}@$ADDRESS exit < /dev/null
+        if [ $? -eq 0 ]; then
+          echo "Successfully added $ADDRESS"
+          break
+        fi
+        n=$((n+1)) 
+        sleep $[ ( $RANDOM % 10 )  + 1 ]s
+      done
+    EOT
+    environment = {
+      ADDRESS = element(flatten(self.ipv4_addresses), 1)
+    }
+    when = create
+  }
+
+}

modules/proxy/output.tf

@@ -0,0 +1,3 @@
+output "proxy_ipv4_address" {
+  value = proxmox_virtual_environment_vm.node.ipv4_addresses[1][0]
+}

modules/proxy/variables.tf

@@ -0,0 +1,14 @@
+variable "ha_proxy_user" {
+  description = "Username for proxy VM"
+  type        = string
+}
+
+variable "DEFAULT_BRIDGE" {
+  description = "Bridge to use when creating VMs in proxmox"
+  type        = string
+}
+
+variable "TARGET_NODE" {
+  description = "Target node name in proxmox"
+  type        = string
+}

varaibles.tf

@@ -69,15 +69,4 @@ variable "worker_config" {
     vcpus   = number
     sockets = number
   })
-}
-
-# HA Proxy config
-variable "ha_proxy_server" {
-  description = "IP address of server running haproxy"
-  type        = string
-}
-
-variable "ha_proxy_user" {
-  description = "User on ha_proxy_server that can modify '/etc/haproxy/haproxy.cfg' and restart haproxy.service"
-  type        = string
 }