Implement ingress setup validation

[draychev]

The GitHub Issue here is to create an osm-health check for ingress.

In the following repo I documented 3 different Ingress scenarios.

1. Check which Ingress helm chart is installed
2. Check for correct annotations on the Ingress resource
3. Check for correct configuration of Contour ConfigMap
4. Check for correct inbound port exclusions
5. Check for existence and validity of mTLS certificate
6. Check for proper MeshConfig
7. Check correctness of IngressBackend configuration
   ... etc.

---

Scenarios

(copied from https://github.com/draychev/osm-benchmarks/tree/main/ingress-benchmarks#readme)

1. Contour + OSM - Contour is given an mTLS certificate to participate in the mesh
   

2. NGINX + OSM - NGINX is given an mTLS cert
   

3. NGINX in the OSM mesh - Nginx is installed in a namespace, which participates in the mesh and is sidecared with an Envoy proxy (one pod 2 proxies - one Nginx, one Envoy)
   

The experiment was ran on 3 unique AKS clusters with the same characteristics.

1. run--ingress--contour.sh
2. run--ingress--nginx.sh
3. run--ingress--nginx-inmesh.sh

---

Here is a Contour setup

#!/bin/bash

set -auexo pipefail

## Just making sure we don't make a mess on the same cluster
kubectx one

OSM_NAMESPACE="${OSM_NAMESPACE:-osm-system}"

BOOKBUYER_NAMESPACE='bookbuyer'
BOOKSTORE_NAMESPACE='bookstore'

# Cleanup
kubectl delete namespace $(kubectl get namespaces --no-headers | awk '{print $1}' | grep -E '^book') --wait || true

kubectl create namespace bookbuyer
kubectl create namespace bookstore

osm namespace add bookbuyer
osm namespace add bookstore

########################################

# Enable SMI mode
echo -e "Enable SMI mode (permissiveTrafficPolicyMode = false"
kubectl patch meshconfig osm-mesh-config \
  --namespace $OSM_NAMESPACE \
  --patch '{"spec":{"traffic":{"enablePermissiveTrafficPolicyMode":false}}}' \
  --type=merge

########################################


./deploy-apps.sh

kubectl annotate \
        service bookstore -n bookstore \
        projectcontour.io/upstream-protocol.tls='14001' \
        --overwrite

./copy-osm-ca-bundle.sh

########################################


# Install Contour Ingress Controller in the defualt namespace
kubectl create namespace ingress-contour || true
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm upgrade \
  --install contour bitnami/contour \
  --namespace ingress-contour \
  --create-namespace \
  --version 4.3.9

if [ $? != 0 ]; then
 echo "Error installing Contour."
 exit 1
fi

# Either patch or edit the config map and add the TLS params below
kubectl patch ConfigMap contour \
        -n ingress-contour \
        -p '{"data":{"contour.yaml":{"tls":"envoy-client-certificate":{"name":"osm-ingress-mtls","namespace":"ingress-contour"}}}}' \
        --type=merge || true

# Here is the Contour ConfigMap
### data:
###   contour.yaml: |
###     accesslog-format: envoy
###     disablePermitInsecure: false
###     envoy-service-name: 'contour-envoy'
###     leaderelection:
###       configmap-namespace: 'ingress-contour'
###     tls:
###       fallback-certificate: null
###       envoy-client-certificate:
###         name: osm-ingress-mtls
###         namespace: ingress-contour

kubectl label namespace ingress-contour openservicemesh.io/monitored-by=osm --overwrite=true

kubectl rollout restart -n ingress-contour deployment contour-contour


# Remove the old cert -- OSM will create a new one
kubectl delete secret -n ingress-contour osm-ingress-mtls || true

kubectl patch MeshConfig \
  osm-mesh-config \
  --namespace $OSM_NAMESPACE \
  --patch '{"spec":{"certificate":{"ingressGateway":{"subjectAltNames":["ingress-contour.ingress-contour.cluster.local"], "validityDuration":"24h", "secret":{"name":"osm-ingress-mtls","namespace":"ingress-contour"}}}}}' \
  --type=merge


kubectl apply -f - <<EOF
---
apiVersion: projectcontour.io/v1
kind: TLSCertificateDelegation
metadata:
  name: ca-secret
  namespace: ingress-contour
spec:
  delegations:
    - secretName: osm-ca-bundle
      targetNamespaces:
      - bookstore
---
apiVersion: projectcontour.io/v1
kind: HTTPProxy
metadata:
  name: bookstore
  namespace: bookstore
spec:
  virtualhost:
    fqdn: osm-bookstore.contoso.com
  routes:
  - services:
    - name: bookstore
      port: 14001
      validation:
        caSecret: osm-ca-bundle
        subjectName: bookstore.bookstore.cluster.local
---
apiVersion: policy.openservicemesh.io/v1alpha1
kind: IngressBackend
metadata:
  name: bookstore
  namespace: bookstore
spec:
  backends:
  - name: bookstore
    port:
      number: 14001
      protocol: https
    tls:
      skipClientCertValidation: false
  sources:
  - kind: Service
    name: contour-envoy
    namespace: ingress-contour
  - kind: AuthenticatedPrincipal
    name: ingress-contour.ingress-contour.cluster.local
EOF

#########################################33

./show-debug.sh

---

Here is NGINX setup

#!/bin/bash

set -auexo pipefail

## Just making sure we don't make a mess on the same cluster
kubectx two

OSM_NAMESPACE="${OSM_NAMESPACE:-osm-system}"

BOOKBUYER_NAMESPACE='bookbuyer'
BOOKSTORE_NAMESPACE='bookstore'

# Cleanup
# kubectl delete namespace $(kubectl get namespaces --no-headers | awk '{print $1}' | grep -E '^book') --wait || true

kubectl create namespace bookbuyer || true
kubectl create namespace bookstore || true

osm namespace add bookbuyer
osm namespace add bookstore

########################################

# Enable SMI mode
echo -e "Enable SMI mode (permissiveTrafficPolicyMode = false"
kubectl patch meshconfig osm-mesh-config \
  --namespace $OSM_NAMESPACE \
  --patch '{"spec":{"traffic":{"enablePermissiveTrafficPolicyMode":false}}}' \
  --type=merge

########################################


# Install Nginx Ingress Controller in the defualt namespace
# helm upgrade \
#   --install ingress-nginx ingress-nginx \
#   --repo https://kubernetes.github.io/ingress-nginx \
#   --namespace ingress-nginx \
#   --create-namespace

kubectl label namespace ingress-nginx openservicemesh.io/monitored-by=osm --overwrite=true


# Remove the old cert -- OSM will create a new one
kubectl delete secret -n ingress-nginx osm-ingress-mtls || true

kubectl patch MeshConfig \
  osm-mesh-config \
  --namespace $OSM_NAMESPACE \
  --patch '{"spec":{"certificate":{"ingressGateway":{"subjectAltNames":["ingress-nginx.ingress-nginx.cluster.local"], "validityDuration":"24h", "secret":{"name":"osm-ingress-mtls","namespace":"ingress-nginx"}}}}}' \
  --type=merge

kubectl rollout restart -n ingress-nginx deployment ingress-nginx-controller

kubectl delete ValidatingWebhookConfiguration ingress-nginx-admission || true

kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: bookstore
  namespace: bookstore
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    # proxy_ssl_name for a service is of the form <service-account>.<namespace>.cluster.local
    nginx.ingress.kubernetes.io/configuration-snippet: |
      proxy_ssl_name "bookstore.bookstore.cluster.local";
    nginx.ingress.kubernetes.io/proxy-ssl-secret: "ingress-nginx/osm-ingress-mtls"
    nginx.ingress.kubernetes.io/proxy-ssl-verify: "on"
spec:
  ingressClassName: nginx
  rules:
  - host: osm-bookstore.contoso.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: bookstore
            port:
              number: 14001
EOF


kubectl apply -f - <<EOF
apiVersion: policy.openservicemesh.io/v1alpha1
kind: IngressBackend
metadata:
  name: bookstore
  namespace: bookstore
spec:
  backends:
  - name: bookstore
    port:
      number: 14001
      protocol: https
    tls:
      skipClientCertValidation: true
  sources:
  - kind: Service
    name: ingress-nginx-controller
    namespace: ingress-nginx
  - kind: AuthenticatedPrincipal
    name: ingress-nginx.ingress-nginx.cluster.local
EOF

#########################################33

./deploy-apps.sh

./show-debug.sh

---

Here is NGINX inmesh

#!/bin/bash

set -auexo pipefail

## Just making sure we don't make a mess on the same cluster
kubectx three

OSM_NAMESPACE="${OSM_NAMESPACE:-osm-system}"

BOOKBUYER_NAMESPACE='bookbuyer'
BOOKSTORE_NAMESPACE='bookstore'

# Cleanup
kubectl delete namespace $(kubectl get namespaces --no-headers | awk '{print $1}' | grep -E '^book') --wait || true

kubectl create namespace bookbuyer
kubectl create namespace bookstore

osm namespace add bookbuyer
osm namespace add bookstore

########################################

# Enable SMI mode
echo -e "Enable SMI mode (permissiveTrafficPolicyMode = false"
kubectl patch meshconfig osm-mesh-config \
  --namespace $OSM_NAMESPACE \
  --patch '{"spec":{"traffic":{"enablePermissiveTrafficPolicyMode":false}}}' \
  --type=merge

########################################

osm namespace remove ingress-nginx

# Install Nginx Ingress Controller in the defualt namespace
helm upgrade \
  --install ingress-nginx ingress-nginx \
  --repo https://kubernetes.github.io/ingress-nginx \
  --namespace ingress-nginx \
  --create-namespace

### Annotate NGINX Ingress Controller with this
### kubectl annotate pod <pod> -n ingress-nginx openservicemesh.io/inbound-port-exclusion-list=80,443,10254
kubectl patch deployment -n ingress-nginx ingress-nginx-controller -p '{"spec":{"template":{"metadata":{"annotations":{"openservicemesh.io/inbound-port-exclusion-list": "80,443,10254"}}}}}'
kubectl patch meshconfig osm-mesh-config -n osm-system -p '{"spec":{"traffic":{"inboundPortExclusionList":[80,443,10254]}}}'  --type=merge
kubectl patch deployment -n ingress-nginx ingress-nginx-controller -p '{"spec":{"template":{"metadata":{"annotations":{"openservicemesh.io/outbound-port-exclusion-list": "80,443,10254"}}}}}'
kubectl patch meshconfig osm-mesh-config -n osm-system -p '{"spec":{"traffic":{"outboundPortExclusionList":[80,443,10254]}}}'  --type=merge

### WE ADD THE NGINX NAMESPACE TO THE MESH
osm namespace add ingress-nginx

kubectl rollout restart -n ingress-nginx deployment ingress-nginx-controller

kubectl delete ValidatingWebhookConfiguration ingress-nginx-admission

sleep 5


kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: bookstore
  namespace: bookstore
  annotations:
    nginx.ingress.kubernetes.io/service-upstream: "true"
    nginx.ingress.kubernetes.io/upstream-vhost: bookstore.bookstore.svc.cluster.local
spec:
  ingressClassName: nginx
  rules:
  - host: osm-bookstore.contoso.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: bookstore
            port:
              number: 14001
EOF

#########################################33

./deploy-apps.sh
./show-debug.sh