chart,salt,tests,docs: Add metallb deployment in MetalK8s deployment

As part of MetalK8s we will use metallb for control plane ingress
if this one is enabled in the Bootstrap Config

Chart get rendered with this command:
```
./charts/render.py metallb --namespace metalk8s-metallb \
  charts/metallb.yaml charts/metallb/ \
  > salt/metalk8s/addons/metallb/deployed/chart.sls
```

NOTE: When we use metallb we do not need to use Nginx Ingress as a
DaemonSet, instead we use a Deployment

Nginx Ingress Control Plane deployment get rendered with this command:
```
./charts/render.py ingress-nginx-control-plane --namespace metalk8s-ingress \
  charts/ingress-nginx-control-plane-deployment.yaml charts/ingress-nginx/ \
  > salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart-deployment.sls
```

Fixes: #2381

buildchain/buildchain/constants.py

@@ -17,6 +17,7 @@
 CMD_WIDTH: int = 14
 
 # URLs of the main container repositories.
+BITNAMI_REPOSITORY: str = "docker.io/bitnami"
 CALICO_REPOSITORY: str = "docker.io/calico"
 COREDNS_REPOSITORY: str = "k8s.gcr.io/coredns"
 COREOS_REPOSITORY: str = "quay.io/coreos"

buildchain/buildchain/image.py

@@ -174,6 +174,7 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage:
 TO_PULL: List[targets.RemoteImage] = []
 
 IMGS_PER_REPOSITORY: Dict[str, List[str]] = {
+    constants.BITNAMI_REPOSITORY: ["metallb-controller", "metallb-speaker"],
     constants.CALICO_REPOSITORY: [
         "calico-node",
         "calico-kube-controllers",

buildchain/buildchain/salt_tree.py

@@ -429,7 +429,15 @@ def _get_parts(self) -> Iterator[str]:
         "salt/metalk8s/addons/nginx-ingress-control-plane/deployed/",
         "chart-daemonset.sls",
     ),
+    Path(
+        "salt/metalk8s/addons/nginx-ingress-control-plane/deployed/",
+        "chart-deployment.sls",
+    ),
     Path("salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/chart.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/config.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/init.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/namespace.sls"),
     Path("salt/metalk8s/beacon/certificates.sls"),
     Path("salt/metalk8s/container-engine/containerd/configured.sls"),
     Path("salt/metalk8s/container-engine/containerd/files/50-metalk8s.conf.j2"),

buildchain/buildchain/versions.py

@@ -175,6 +175,16 @@ def _version_prefix(version: str, prefix: str = "v") -> str:
         version="v1.1.2",
         digest="sha256:22fbde17ab647ddf89841e5e464464eece111402b7d599882c2a3393bc0d2810",
     ),
+    Image(
+        name="metallb-controller",
+        version="0.9.6-debian-10-r52",
+        digest="sha256:a493e311beb663c7282fe6c3712899ab6fb7bd0ad4a38388ea1b97d7d735ff8a",
+    ),
+    Image(
+        name="metallb-speaker",
+        version="0.9.6-debian-10-r54",
+        digest="sha256:27f94679069f184d9bc67f6d5eccfc961be7588a73a066a8a4f87b6b3ef75614",
+    ),
     Image(
         name="pause",
         version="3.2",

charts/ingress-nginx-control-plane-deployment.yaml

@@ -0,0 +1,59 @@
+controller:
+  image:
+    digest: null
+    repository: '__image__(nginx-ingress-controller)'
+
+  defaultBackendService: 'metalk8s-ingress/nginx-ingress-default-backend'
+
+  electionID: ingress-control-plane-controller-leader
+
+  ingressClass: nginx-control-plane
+
+  admissionWebhooks:
+    enabled: false
+
+  kind: Deployment
+
+  replicaCount: 2
+
+  minAvailable: 0
+
+  updateStrategy:
+    type: RollingUpdate
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''
+
+  service:
+    loadBalancerIP: '__var__(salt.metalk8s_network.get_control_plane_ingress_ip())'
+    externalTrafficPolicy: Local
+
+    enableHttp: false
+
+    ports:
+      https: 8443
+
+  extraArgs:
+    default-ssl-certificate: "metalk8s-ingress/ingress-control-plane-default-certificate"
+    metrics-per-host: false
+
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      additionalLabels:
+        metalk8s.scality.com/monitor: ''
+
+defaultBackend:
+  enabled: false

charts/metallb.yaml

@@ -0,0 +1,39 @@
+existingConfigMap: metallb-config
+
+controller:
+  image:
+    registry: null
+    repository: '__image__(metallb-controller)'
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+speaker:
+  image:
+    registry: null
+    repository: '__image__(metallb-speaker)'
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"

docs/installation/bootstrap.rst

@@ -48,6 +48,9 @@ Configuration
         controlPlane:
           cidr: <CIDR-notation>
           ingressIP: <IP-for-ingress>
+          metalLB:
+            enabled: true
+            config: <metallb-config>
         workloadPlane:
           cidr: <CIDR-notation>
           mtu: <network-MTU>
@@ -90,6 +93,18 @@ notation for it's various subfields.
       that if you lose the Bootstrap node, you no longer have access to any
       control plane component).
 
+      This ``ingressIP`` can be managed by MetalK8s directly if it's possible
+      in your environment, to do so we use
+      `MetalLB <https://metallb.universe.tf/>`_ that allow to manage this
+      Virtual IP directly on Layer2 using only
+      `ARP <https://en.wikipedia.org/wiki/Address_Resolution_Protocol>`_
+      requests, this is the default configuration deployed in MetalK8s, but
+      you can also override this default configuration so that you can use
+      `BGP configuration <https://metallb.universe.tf/configuration/#bgp-configuration>`_
+      if you want to integrate it with your BGP router.
+      When MetalLB is enabled ``ingressIP`` is mandatory, if you use Layer2
+      configuration you can ommit the ``config`` field of MetalLB
+
       For ``workloadPlane`` entry an
       `MTU <https://en.wikipedia.org/wiki/Maximum_transmission_unit>`_ can
       also be provided, this MTU value should be the lowest MTU value accross

docs/operation/changing_control_plane_ingress_ip.rst

@@ -1,8 +1,11 @@
 Changing the Control Plane Ingress IP
 =====================================
 
-#. On the bootstrap node, update the ``ingressIP`` field from
-   ``networks.controlPlane`` in the Bootstrap configuration file.
+This procedure allow to change the Control Plane Ingress IP and also to move
+from MetalLB configuration to non-MetalLB configuration and the oposite.
+
+#. On the bootstrap node, update the ``ingressIP`` and/or ``metalLB`` field
+   from ``networks.controlPlane`` in the Bootstrap configuration file.
    (refer to :ref:`Bootstrap Configuration<Bootstrap Configuration>`)
 
 #. Refresh the pillar.
@@ -18,6 +21,14 @@ Changing the Control Plane Ingress IP
       $ salt-call metalk8s_network.get_control_plane_ingress_ip
       local:
           <my-new-ip>
+      $ salt-call pillar.get networks:control_plane
+      local:
+        ----------
+        cidr:
+          - <control-plane-cidr>
+        ingressIP:
+          <my-new-ip>
+        metalLB: <my-new-metallb-config>
 
 #. On the bootstrap node, reconfigure ApiServer:
 

eve/main.yml

@@ -356,6 +356,9 @@ models:
         networks:
           controlPlane:
             cidr: 192.168.1.0/24
+            metalLB:
+              enabled: true
+            ingressIP: 192.168.1.254
           workloadPlane:
             cidr: 192.168.2.0/24
         ca:
@@ -480,7 +483,7 @@ models:
       name: Run UI tests on Bastion
       env: &_env_bastion_ui_tests
         TEST_FILTER: "e2e"
-        TARGET_URL: "https://%(prop:bootstrap_control_plane_ip)s:8443"
+        TARGET_URL: "https://192.168.1.254:8443"
       command: >
         ssh -F ssh_config bastion --
         "cd metalk8s/ui &&

salt/_pillar/metalk8s.py

@@ -1,6 +1,7 @@
 import logging
 from collections import Mapping
 
+import salt.utils.dictupdate
 import salt.utils.files
 import salt.utils.yaml
 
@@ -90,6 +91,33 @@ def _load_networks(config_data):
             if not isinstance(networks_data[net]["cidr"], list):
                 networks_data[net]["cidr"] = [networks_data[net]["cidr"]]
 
+    # MetalLB disabled by default
+    networks_data["controlPlane"].setdefault("metalLB", {}).setdefault("enabled", False)
+
+    if networks_data["controlPlane"]["metalLB"]["enabled"]:
+        if not networks_data["controlPlane"].get("ingressIP"):
+            errors.append(
+                "'ingressIP' for 'controlPlane' network is mandatory when 'metalLB'"
+                "is enabled"
+            )
+        else:
+            address_pools = (
+                networks_data["controlPlane"]["metalLB"]
+                .setdefault("config", {})
+                .setdefault("address-pools", [])
+            )
+            if not address_pools:
+                address_pools.append({})
+
+            address_pools[0].setdefault("name", "ingress-ip")
+            address_pools[0].setdefault("protocol", "layer2")
+
+            # Enfore address to Ingress IP
+            address_pools[0]["addresses"] = [
+                "{}/32".format(networks_data["controlPlane"]["ingressIP"])
+            ]
+            address_pools[0]["auto-assign"] = False
+
     if errors:
         return __utils__["pillar_utils.errors_to_dict"](errors)