Terraform rewrite all packend pools configured by ingress

[ykruchko]

In second deploy terraform delete all configured backends, rules and settings made by ingress-appgw-deployment.

Terrform:
resource "azurerm_public_ip" "aks-ingress" {
name = "${var.workspace}-aks-ingress"
resource_group_name = "${var.workspace}-aks"
location = azurerm_resource_group.rg.location
allocation_method = "Static"
sku = "Standard"
}

resource "azurerm_application_gateway" "aks-ingress" {
location = azurerm_resource_group.rg.location
name = "${var.workspace}-aks-ingress"
resource_group_name = "${var.workspace}-aks"

backend_address_pool {
name = "default"
}
backend_http_settings {
cookie_based_affinity = "Disabled"
name = "default"
port = 80
protocol = "Http"
path = "/"
}
frontend_ip_configuration {
name = azurerm_public_ip.aks-ingress.name
public_ip_address_id = azurerm_public_ip.aks-ingress.id
}
frontend_port {
name = "default"
port = 80
}
gateway_ip_configuration {
name = "public"
subnet_id = azurerm_subnet.public.id
}
http_listener {
frontend_ip_configuration_name = azurerm_public_ip.aks-ingress.name
frontend_port_name = "default"
name = "default"
protocol = "Http"
}

request_routing_rule {
http_listener_name = "default"
name = "default"
rule_type = "Basic"
backend_http_settings_name = "default"
backend_address_pool_name = "default"
}

sku {
name = "WAF_v2"
tier = "WAF_v2"
capacity = 1
}
}

resource "azurerm_kubernetes_cluster" "aks" {
name = var.workspace
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
dns_prefix = var.workspace
private_cluster_enabled = false
node_resource_group = "${var.workspace}-aks"

identity {
type = "SystemAssigned"
}

default_node_pool{
name = "default"
vm_size = var.kube-node-vm-size
node_count = var.kube-pool-node-count
vnet_subnet_id = azurerm_subnet.private.id
}

network_profile {
network_plugin = "azure"
network_policy = "azure"
}
addon_profile {
ingress_application_gateway {
enabled = true
gateway_id = azurerm_application_gateway.aks-ingress.id
}
}
tags = {
Environment = var.workspace
}
depends_on = [
azurerm_subnet.private,
azurerm_resource_group.rg
]
}

resource "kubernetes_ingress" "ingress" {
metadata {
name = "ingress"
namespace = kubernetes_namespace.ns.metadata.0.name
annotations = {
"kubernetes.io/ingress.class" = "azure/application-gateway"
"appgw.ingress.kubernetes.io/ssl-redirect" = "true"
"appgw.ingress.kubernetes.io/waf-policy-for-path" = azurerm_web_application_firewall_policy.ingress-ssi.id
}
}
spec {
tls {
secret_name = kubernetes_secret.ssl.metadata.0.name
hosts = [var.domain]
}
rule {
host = var.domain
http {
path {
backend {
service_name = kubernetes_service.app.metadata.0.name
service_port = kubernetes_service.app.spec.0.port.0.port
}
path = "/"
}
backend {
service_name = kubernetes_service.app2.metadata.0.name
service_port = kubernetes_service.app2.spec.0.port.0.port
}
path = "/app2/*"
}
}
...

Terraform plan
resource "azurerm_application_gateway" "aks-ingress" {
id = "/subscriptions/000000/resourceGroups/dev-aks/providers/Microsoft.Network/applicationGateways/dev-aks-ingress"
name = "dev-aks-ingress"
~ tags = {
- "managed-by-k8s-ingress" = "1.4.0/0e9dc17c/2021-02-26-03:29T+0000" -> null
}
# (4 unchanged attributes hidden)
~ backend_address_pool {
id = "/subscriptions/000000/resourceGroups/dev-aks/providers/Microsoft.Network/applicationGateways/dev-aks-ingress/backendAddressPools/defaultaddresspool"
~ name = "defaultaddresspool" -> "default"
# (2 unchanged attributes hidden)
}
- backend_address_pool {
- fqdns = [] -> null
- id = "/subscriptions/000000/resourceGroups/dev-aks/providers/Microsoft.Network/applicationGateways/dev-aks-ingress/backendAddressPools/pool-dev-app-8080-bp-8080" -> null
- ip_addresses = [
- "192.168.150.26",
] -> null
....

[ykruchko]

My workaround

ingress_pods=`kubectl -n kube-system  get pod  | awk '{if ($1 ~ "ingress-appgw-deployment-") print $1}'`
kubectl -n kube-system delete pod $ingress_pods