diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index a311aad14b..1ae8ab8587 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -545,6 +545,7 @@ def _get_parts(self) -> Iterator[str]: Path("salt/metalk8s/orchestrate/downgrade/precheck.sls"), Path("salt/metalk8s/orchestrate/downgrade/pre.sls"), Path("salt/metalk8s/orchestrate/downgrade/post.sls"), + Path("salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls"), Path("salt/metalk8s/orchestrate/upgrade/init.sls"), Path("salt/metalk8s/orchestrate/upgrade/precheck.sls"), Path("salt/metalk8s/orchestrate/upgrade/pre.sls"), diff --git a/docs/operation/changing_control_plane_ingress_ip.rst b/docs/operation/changing_control_plane_ingress_ip.rst new file mode 100644 index 0000000000..0a586272f8 --- /dev/null +++ b/docs/operation/changing_control_plane_ingress_ip.rst @@ -0,0 +1,44 @@ +Changing the Control Plane Ingress IP +===================================== + +#. On the Bootstrap node, update the ``ip`` field from + ``networks.controlPlane.ingress`` in the Bootstrap configuration file. + (refer to :ref:`Bootstrap Configuration`) + +#. Refresh the pillar. + + .. code-block:: console + + $ salt-call saltutil.refresh_pillar wait=True + +#. Check that the change is taken into account. + + .. code-block:: console + + $ salt-call metalk8s_network.get_control_plane_ingress_ip + local: + + +#. On the Bootstrap node, reconfigure apiServer: + + .. parsed-literal:: + + $ salt-call state.sls \\ + metalk8s.kubernetes.apiserver \\ + saltenv=metalk8s-|version| + +#. Reconfigure Control Plane components: + + .. parsed-literal:: + + $ kubectl exec -n kube-system -c salt-master \\ + --kubeconfig=/etc/kubernetes/admin.conf \\ + $(kubectl --kubeconfig=/etc/kubernetes/admin.conf get pod \\ + -l "app.kubernetes.io/name=salt-master" \\ + --namespace=kube-system -o jsonpath='{.items[0].metadata.name}') \\ + -- salt-run state.orchestrate \\ + metalk8s.orchestrate.update-control-plane-ingress-ip \\ + saltenv=metalk8s-|version| + +#. You can :ref:`access the MetalK8s GUI ` + using this new IP. diff --git a/docs/operation/index.rst b/docs/operation/index.rst index a08c430b6e..e056cf3fdd 100644 --- a/docs/operation/index.rst +++ b/docs/operation/index.rst @@ -19,6 +19,7 @@ do not have a working MetalK8s_ setup. disaster_recovery/index solutions changing_node_hostname + changing_control_plane_ingress_ip metalk8s-utils registry_ha listening_processes diff --git a/salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls b/salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls new file mode 100644 index 0000000000..8bde7b3253 --- /dev/null +++ b/salt/metalk8s/orchestrate/update-control-plane-ingress-ip.sls @@ -0,0 +1,57 @@ +{%- set bootstrap_node = salt.metalk8s.minions_by_role('bootstrap') | first %} +Check pillar content on {{ bootstrap_node }}: + salt.function: + - name: metalk8s.check_pillar_keys + - tgt: {{ bootstrap_node }} + - kwarg: + keys: + - metalk8s.endpoints.repositories + raise_error: False + - retry: + attempts: 5 + +Regenerate Control Plane Ingress cert on {{ bootstrap_node }}: + salt.state: + - tgt: {{ bootstrap_node }} + - sls: + - metalk8s.addons.nginx-ingress-control-plane.certs + - saltenv: {{ saltenv }} + - require: + - salt: Check pillar content on {{ bootstrap_node }} + +Reconfigure Control Plane Ingress: + salt.runner: + - name: state.orchestrate + - mods: + - metalk8s.addons.nginx-ingress-control-plane.deployed + - saltenv: {{ saltenv }} + - require: + - salt: Regenerate Control Plane Ingress cert on {{ bootstrap_node }} + +Reconfigure Control Plane components: + salt.runner: + - name: state.orchestrate + - mods: + - metalk8s.addons.dex.deployed + - metalk8s.addons.prometheus-operator.deployed + - metalk8s.addons.ui.deployed + - saltenv: {{ saltenv }} + - require: + - salt: Reconfigure Control Plane Ingress + +{%- set master_nodes = salt.metalk8s.minions_by_role('master') %} +{%- for node in master_nodes | sort %} + +Reconfigure apiserver on {{ node }}: + salt.state: + - tgt: {{ node }} + - sls: + - metalk8s.kubernetes.apiserver + - saltenv: {{ saltenv }} + - require: + - salt: Reconfigure Control Plane components + {%- if loop.previtem is defined %} + - salt: Reconfigure apiserver on {{ loop.previtem }} + {%- endif %} + +{%- endfor %} diff --git a/tests/post/features/ingress.feature b/tests/post/features/ingress.feature index 3a06c69424..2c834903e0 100644 --- a/tests/post/features/ingress.feature +++ b/tests/post/features/ingress.feature @@ -18,3 +18,13 @@ Feature: Ingress And the node control-plane IP is not equal to its workload-plane IP When we perform an HTTP request on port 80 on a control-plane IP Then the server should not respond + + Scenario: Change Control Plane Ingress IP to node-1 IP + Given the Kubernetes API is available + And we are on a multi node cluster + And pods with label 'app.kubernetes.io/name=ingress-nginx' are 'Ready' + When we update control plane ingress IP to node 'node-1' IP + And we wait for the rollout of 'daemonset/ingress-nginx-control-plane-controller' in namespace 'metalk8s-ingress' to complete + And we wait for the rollout of 'deploy/dex' in namespace 'metalk8s-auth' to complete + Then the control plane ingress IP is equal to node 'node-1' IP + And we are able to login to Dex as 'admin@metalk8s.invalid' using password 'password' diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index 4cf2dc3cc5..b930cd04ee 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -1,8 +1,11 @@ +import json +import os import requests import requests.exceptions import pytest from pytest_bdd import given, parsers, scenario, then, when +import testinfra from tests import utils @@ -22,11 +25,30 @@ def test_access_http_services_on_control_plane_ip(host): pass +@scenario("../features/ingress.feature", "Change Control Plane Ingress IP to node-1 IP") +def test_change_cp_ingress_ip(host, teardown): + pass + + @pytest.fixture(scope="function") def context(): return {} +@pytest.fixture +def teardown(context, host, ssh_config, version): + yield + if "bootstrap_to_restore" in context: + with host.sudo(): + host.check_output( + "cp {} /etc/metalk8s/bootstrap.yaml".format( + context["bootstrap_to_restore"] + ) + ) + + re_configure_cp_ingress(host, version, ssh_config) + + @given("the node control-plane IP is not equal to its workload-plane IP") def node_control_plane_ip_is_not_equal_to_its_workload_plane_ip(host): data = utils.get_grain(host, "metalk8s") @@ -67,6 +89,17 @@ def perform_request(host, context, protocol, port, plane): context["exception"] = exc +@when(parsers.parse("we update control plane ingress IP to node '{node_name}' IP")) +def update_cp_ingress_ip(host, context, ssh_config, version, node_name): + node = testinfra.get_host(node_name, ssh_config=ssh_config) + ip = utils.get_grain(node, "metalk8s:control_plane_ip") + + bootstrap_patch = {"networks": {"controlPlane": {"ingress": {"ip": ip}}}} + + patch_bootstrap_config(context, host, bootstrap_patch) + re_configure_cp_ingress(host, version, ssh_config) + + @then( parsers.re(r"the server returns (?P\d+) '(?P.+)'"), converters=dict(status_code=int), @@ -82,3 +115,48 @@ def server_returns(host, context, status_code, reason): def server_does_not_respond(host, context): assert "exception" in context assert isinstance(context["exception"], requests.exceptions.ConnectionError) + + +@then(parsers.parse("the control plane ingress IP is equal to node '{node_name}' IP")) +def check_cp_ingress_node_ip(control_plane_ingress_ip, node_name, ssh_config): + node = testinfra.get_host(node_name, ssh_config=ssh_config) + ip = utils.get_grain(node, "metalk8s:control_plane_ip") + + assert control_plane_ingress_ip == ip + + +def patch_bootstrap_config(context, host, patch): + with host.sudo(): + cmd_ret = host.check_output("salt-call --out json --local temp.dir") + + tmp_dir = json.loads(cmd_ret)["local"] + + with host.sudo(): + host.check_output("cp /etc/metalk8s/bootstrap.yaml {}".format(tmp_dir)) + + context["bootstrap_to_restore"] = os.path.join(tmp_dir, "bootstrap.yaml") + + with host.sudo(): + host.check_output( + "salt-call --local --retcode-passthrough state.single " + "file.serialize /etc/metalk8s/bootstrap.yaml " + "dataset='{}' " + "merge_if_exists=True".format(json.dumps(patch)) + ) + + +def re_configure_cp_ingress(host, version, ssh_config): + with host.sudo(): + host.check_output( + "salt-call --retcode-passthrough state.sls " + "metalk8s.kubernetes.apiserver saltenv=metalk8s-{}".format(version) + ) + + command = [ + "salt-run", + "state.orchestrate", + "metalk8s.orchestrate.update-control-plane-ingress-ip", + "saltenv=metalk8s-{}".format(version), + ] + + utils.run_salt_command(host, command, ssh_config)