WIP - updates Azure Terraform example files to allow users to create a cluster within an internal network only

examples/terraform/azure/README.md

@@ -47,6 +47,9 @@ No modules.
 | [azurerm_virtual_network.vpc](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource |
 | [time_sleep.wait_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
 | [azurerm_public_ip.control_plane](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/public_ip) | data source |
+| [azurerm_network_interface.lb_vm_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface) | resource |
+| [azurerm_virtual_machine.lb_vm](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_machine) | resource |
+| [null_resource.lb_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 
 ## Inputs
 
@@ -65,8 +68,10 @@ No modules.
 | <a name="input_initial_machinedeployment_operating_system_profile"></a> [initial\_machinedeployment\_operating\_system\_profile](#input\_initial\_machinedeployment\_operating\_system\_profile) | Name of operating system profile for MachineDeployments, only applicable if operating-system-manager addon is enabled.<br>If not specified, the default value will be added by machine-controller addon. | `string` | `""` | no |
 | <a name="input_initial_machinedeployment_replicas"></a> [initial\_machinedeployment\_replicas](#input\_initial\_machinedeployment\_replicas) | Number of replicas per MachineDeployment | `number` | `2` | no |
 | <a name="input_ip_sku"></a> [ip\_sku](#input\_ip\_sku) | SKU to use for IP addresses | `string` | `"Basic"` | no |
+| <a name="input_lb_vm_size"></a> [lb_vm_size](#input\_lb_vm_size) | VM Size for Load Balancer machines | `string` | `"Standard_B2s"` | no |
 | <a name="input_location"></a> [location](#input\_location) | Azure datacenter to use | `string` | `"westeurope"` | no |
 | <a name="input_os"></a> [os](#input\_os) | Operating System to use for finding image reference and in MachineDeployment | `string` | `"ubuntu"` | no |
+| <a name="input_private_networking_only"></a> [private_networking_only](#input\_private_networking_only) | Enable this to use private networking only | `bool` | `"false"` | no
 | <a name="input_rhsm_offline_token"></a> [rhsm\_offline\_token](#input\_rhsm\_offline\_token) | RHSM offline token | `string` | `""` | no |
 | <a name="input_rhsm_password"></a> [rhsm\_password](#input\_rhsm\_password) | RHSM password | `string` | `""` | no |
 | <a name="input_rhsm_username"></a> [rhsm\_username](#input\_rhsm\_username) | RHSM username | `string` | `""` | no |

examples/terraform/azure/etc_gobetween.tpl

@@ -0,0 +1,26 @@
+[api]
+enabled = false
+
+[servers.default]
+protocol = "tcp"
+bind = "0.0.0.0:6443"
+balance = "roundrobin"
+max_connections = 10000
+client_idle_timeout = "10m"
+backend_idle_timeout = "10m"
+backend_connection_timeout = "2s"
+
+[servers.default.discovery]
+kind = "static"
+static_list = [
+    %{ for target in lb_targets ~}
+    "${target}:6443",
+    %{ endfor ~}
+]
+
+[servers.default.healthcheck]
+kind = "ping"
+interval = "10s"
+timeout = "2s"
+fails = 2
+passes = 1

examples/terraform/azure/gobetween.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The KubeOne Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is mostly used in CI
+# It installs dependencies and starts the tests
+
+set -euf -o pipefail
+
+GOBETWEEN_VERSION=0.7.0
+
+noop() { : "didn't detected package manager, noop"; }
+
+PKG_MANAGER="noop"
+
+[ "$(command -v yum)" ] && PKG_MANAGER=yum
+[ "$(command -v apt-get)" ] && PKG_MANAGER=apt-get
+
+sudo ${PKG_MANAGER} install tar -y
+
+mkdir -p /tmp/gobetween
+cd /tmp/gobetween
+curl -L -o gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz \
+    https://github.com/yyyar/gobetween/releases/download/${GOBETWEEN_VERSION}/gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz
+tar xvf gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz
+sudo mkdir -p /opt/bin
+sudo mv gobetween /opt/bin/gobetween
+sudo chown root:root /opt/bin/gobetween
+
+cat <<EOF | sudo tee /etc/systemd/system/gobetween.service
+[Unit]
+Description=Gobetween - modern LB for cloud era
+Documentation=https://github.com/yyyar/gobetween/wiki
+After=network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=simple
+ExecStart=/opt/bin/gobetween -c /etc/gobetween.toml
+PrivateTmp=true
+User=nobody
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+sudo systemctl daemon-reload
+sudo systemctl enable gobetween.service
+sudo systemctl start gobetween.service

examples/terraform/azure/main.tf

@@ -20,14 +20,19 @@ provider "azurerm" {
 }
 
 locals {
-  kubeapi_endpoint                   = var.disable_kubeapi_loadbalancer ? azurerm_network_interface.control_plane.0.private_ip_address : azurerm_public_ip.lbip.0.ip_address
+  kubeapi_endpoint                   = var.disable_kubeapi_loadbalancer ? azurerm_network_interface.control_plane.0.private_ip_address : var.private_networking_only ? azurerm_lb.lb.0.private_ip_address : azurerm_public_ip.lbip.0.ip_address
+  rendered_lb_config                 = var.private_networking_only ? templatefile("./etc_gobetween.tpl", {
+    lb_targets                       = azurerm_network_interface.control_plane.*.private_ip_address,
+  }) : null
   loadbalancer_count                 = var.disable_kubeapi_loadbalancer ? 0 : 1
   nic_address_pool_association_count = local.loadbalancer_count > 0 ? var.control_plane_vm_count : 0
   worker_os                          = var.worker_os == "" ? var.image_references[var.os].worker_os : var.worker_os
   ssh_username                       = var.ssh_username == "" ? var.image_references[var.os].ssh_username : var.ssh_username
 
-  cluster_autoscaler_min_replicas = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas
-  cluster_autoscaler_max_replicas = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas
+  cluster_autoscaler_min_replicas    = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas
+  cluster_autoscaler_max_replicas    = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas
+
+  lb_vm_count                        = var.private_networking_only ? 2 : 0
 }
 
 provider "time" {
@@ -139,7 +144,7 @@ resource "azurerm_network_security_group" "sg" {
 }
 
 resource "azurerm_public_ip" "lbip" {
-  count = local.loadbalancer_count
+  count = var.private_networking_only ? 0 : local.loadbalancer_count
 
   name                = "${var.cluster_name}-lbip"
   location            = var.location
@@ -154,7 +159,7 @@ resource "azurerm_public_ip" "lbip" {
 }
 
 resource "azurerm_public_ip" "control_plane" {
-  count = var.control_plane_vm_count
+  count = var.private_networking_only ? 0 : var.control_plane_vm_count
 
   name                = "${var.cluster_name}-cp-${count.index}"
   location            = var.location
@@ -175,9 +180,21 @@ resource "azurerm_lb" "lb" {
   name                = "kubernetes"
   location            = var.location
 
-  frontend_ip_configuration {
-    name                 = "KubeApi"
-    public_ip_address_id = azurerm_public_ip.lbip.0.id
+  dynamic "frontend_ip_configuration" {
+    for_each = var.private_networking_only ? [1] : []
+    content {
+      name                          = "KubeApi"
+      subnet_id                     = azurerm_subnet.subnet.id
+      private_ip_address_allocation = "Dynamic"
+    }
+  }
+
+  dynamic "frontend_ip_configuration" {
+    for_each = var.private_networking_only ? [] : [1]
+    content {
+      name                 = "KubeApi"
+      public_ip_address_id = azurerm_public_ip.lbip.0.id
+    }
   }
 
   tags = {
@@ -227,11 +244,23 @@ resource "azurerm_network_interface" "control_plane" {
   location            = var.location
   resource_group_name = azurerm_resource_group.rg.name
 
-  ip_configuration {
-    name                          = "${var.cluster_name}-cp-${count.index}"
-    subnet_id                     = azurerm_subnet.subnet.id
-    private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = element(azurerm_public_ip.control_plane.*.id, count.index)
+  dynamic "ip_configuration" {
+    for_each = var.private_networking_only ? [] : [1]
+    content {
+      name                          = "${var.cluster_name}-cp-${count.index}"
+      subnet_id                     = azurerm_subnet.subnet.id
+      private_ip_address_allocation = "Dynamic"
+      public_ip_address_id          = element(azurerm_public_ip.control_plane.*.id, count.index)
+    }
+  }
+
+  dynamic "ip_configuration" {
+    for_each = var.private_networking_only ? [1] : []
+    content {
+      name                          = "${var.cluster_name}-cp-${count.index}"
+      subnet_id                     = azurerm_subnet.subnet.id
+      private_ip_address_allocation = "Dynamic"
+    }
   }
 }
 
@@ -314,7 +343,116 @@ data "azurerm_public_ip" "control_plane" {
   depends_on = [
     time_sleep.wait_30_seconds
   ]
-  count               = var.control_plane_vm_count
+  count               = var.private_networking_only ? 0 : var.control_plane_vm_count
   name                = "${var.cluster_name}-cp-${count.index}"
   resource_group_name = azurerm_resource_group.rg.name
 }
+
+resource "azurerm_network_interface" "lb_vm_network" {
+  depends_on = [
+    azurerm_lb.lb,
+  ]
+  count = var.private_networking_only ? local.lb_vm_count : 0
+
+  name                = "${var.cluster_name}-lb-${count.index}"
+  location            = var.location
+  resource_group_name = azurerm_resource_group.rg.name
+  ip_configuration {
+    name                          = "${var.cluster_name}-lb-${count.index}"
+    subnet_id                     = azurerm_subnet.subnet.id
+    private_ip_address_allocation = "Dynamic"
+  }
+
+}
+resource "azurerm_virtual_machine" "lb_vm" {
+  depends_on = [
+    azurerm_network_interface.lb_vm_network,
+    azurerm_virtual_machine.control_plane
+  ]
+  count = var.private_networking_only ? local.lb_vm_count : 0
+
+  name                             = "${var.cluster_name}-lb-${count.index}"
+  location                         = var.location
+  resource_group_name              = azurerm_resource_group.rg.name
+  availability_set_id              = azurerm_availability_set.avset.id
+  vm_size                          = var.lb_vm_size
+  network_interface_ids            = [element(azurerm_network_interface.lb_vm_network.*.id, count.index)]
+  delete_os_disk_on_termination    = true
+  delete_data_disks_on_termination = true
+
+  dynamic "plan" {
+    for_each = var.image_references[var.os].plan
+
+    content {
+      name      = plan.value["name"]
+      publisher = plan.value["publisher"]
+      product   = plan.value["product"]
+    }
+  }
+  storage_image_reference {
+    publisher = var.image_references[var.os].image.publisher
+    offer     = var.image_references[var.os].image.offer
+    sku       = var.image_references[var.os].image.sku
+    version   = var.image_references[var.os].image.version
+  }
+
+  storage_os_disk {
+    name              = "${var.cluster_name}-lb-${count.index}"
+    caching           = "ReadWrite"
+    create_option     = "FromImage"
+    managed_disk_type = "Standard_LRS"
+  }
+
+  os_profile {
+    computer_name  = "${var.cluster_name}-lb-${count.index}"
+    admin_username = local.ssh_username
+    custom_data    = file("gobetween.sh")
+  }
+
+  os_profile_linux_config {
+    disable_password_authentication = true
+
+    ssh_keys {
+      key_data = file(var.ssh_public_key_file)
+      path     = "/home/${local.ssh_username}/.ssh/authorized_keys"
+    }
+  }
+
+  tags = {
+    environment = "kubeone"
+    cluster     = var.cluster_name
+  }
+
+}
+
+
+resource "null_resource" "lb_config" {
+  depends_on = [
+    azurerm_virtual_machine.lb_vm
+  ]
+  triggers = {
+    lb_instance_ids      = join(",", azurerm_virtual_machine.lb_vm.*.id)
+    cluster_instance_ids = join(",", azurerm_virtual_machine.control_plane.*.id)
+    config               = local.rendered_lb_config
+  }
+
+  count = var.private_networking_only ? local.lb_vm_count : 0
+  connection {
+    type = "ssh"
+    user = var.ssh_username
+    private_key = file(var.ssh_private_key_file)
+    host = element(azurerm_network_interface.lb_vm_network.*.private_ip_address, count.index)
+  }
+
+  provisioner "file" {
+    content     = local.rendered_lb_config
+    destination = "/tmp/gobetween.toml"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mv /tmp/gobetween.toml /etc/gobetween.toml",
+      "sudo systemctl restart gobetween",
+    ]
+  }
+}

examples/terraform/azure/output.tf

@@ -95,7 +95,7 @@ output "kubeone_workers" {
           availabilitySet = azurerm_availability_set.avset_workers.name
           # assignAvailabilitySet = true/false
           securityGroupName = azurerm_network_security_group.sg.name
-          assignPublicIP    = true
+          assignPublicIP    = var.private_networking_only ? false : true
           imageReference    = var.os != "rhel" ? var.image_references[var.os].image : null
           imagePlan         = length(var.image_references[var.os].plan) > 0 && var.os != "rhel" ? var.image_references[var.os].plan[0] : null
           # Zones (optional)

examples/terraform/azure/variables.tf

@@ -24,6 +24,12 @@ variable "cluster_name" {
   }
 }
 
+variable "private_networking_only" {
+  description = "Enable this to use private networking only"
+  default     = false
+  type        = bool
+}
+
 variable "apiserver_alternative_names" {
   description = "subject alternative names for the API Server signing cert."
   default     = []
@@ -281,3 +287,9 @@ variable "disable_auto_update" {
   type        = bool
   default     = false
 }
+
+variable "lb_vm_size" {
+  description = "VM Size for Load Balancer machines"
+  type        = string
+  default = "Standard_B2s"
+}