From 301006fbb7596575f8c49059aecd857cdb7896d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 18 Jun 2024 19:02:06 +0200 Subject: [PATCH 01/77] Add dataclasses for queries to and responses from the Kea REST api --- python/nav/kea_stats.py | 78 +++++++++++++++++++++++++++++++ requirements/base.txt | 2 + tests/unittests/kea_stats_test.py | 47 +++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 python/nav/kea_stats.py create mode 100644 tests/unittests/kea_stats_test.py diff --git a/python/nav/kea_stats.py b/python/nav/kea_stats.py new file mode 100644 index 0000000000..34e971a7dd --- /dev/null +++ b/python/nav/kea_stats.py @@ -0,0 +1,78 @@ +""" +Functions for querying the Kea Control Agent for statistics from Kea DHCP +servers. + +Stork (https://gitlab.isc.org/isc-projects/stork) is used as a guiding +implementation for interacting with the Kea Control Agent. See also the Kea +Control Agent documentation +(https://kea.readthedocs.io/en/kea-2.6.0/arm/agent.html). +""" +from dataclasses import dataclass, asdict +from enum import Enum +import logging +import requests +from requests.exceptions import JSONDecodeError, HTTPError +from typing import Union + +logger = logging.getLogger(__name__) + +class KeaStatus(Enum): + # Successful operation. + SUCCESS = 0 + # General failure. + ERROR = 1 + # Command is not supported. + UNSUPPORTED = 2 + # Successful operation, but failed to produce any results. + EMPTY = 3 + # Unsuccessful operation due to a conflict between the command arguments and the server state. + CONFLICT = 4 + +@dataclass +class KeaResponse: + """ + Class for defining a REST response on a REST query sent to a kea-ctrl-agent + process. + """ + result: int + text: str + arguments: dict[str: Union[str, int]] + service: str + +@dataclass +class KeaQuery: + """ + Class for defining a REST query to be sent to a kea-ctrl-agent + process. + """ + command: str + arguments: dict[str: Union[str, int]] + services: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. + + def send(self, address) -> list[KeaResponse]: + """ + Send this query to a kea-ctrl-agent located at address + """ + logger.debug("KeaQuery.send: sending request to %s with query %r", address, self) + try: + r = requests.post(address, data=asdict(self)) + except HTTPError as err: + logger.error("KeaQuery.send: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) + raise err + + try: + json = r.json() + except JSONDecodeError as err: + logger.error("KeaQuery.send: expected json from %s, got %s", address, r.text) + raise err + + responses = [] + for obj in json: + response = KeaResponse( + obj.get("result", KeaStatus.ERROR), + obj.get("text", ""), + obj.get("arguments", {}), + obj.get("service", ""), + ) + responses.append(response) + return responses diff --git a/requirements/base.txt b/requirements/base.txt index e95e419e3d..088e566cde 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -47,3 +47,5 @@ git+https://github.com/Uninett/drf-oidc-auth@v4.0#egg=drf-oidc-auth PyOpenSSL==23.3.0 # service-identity is required to make TLS communication libraries shut up about potential MITM attacks service-identity==21.1.0 + +requests diff --git a/tests/unittests/kea_stats_test.py b/tests/unittests/kea_stats_test.py new file mode 100644 index 0000000000..decc7e883a --- /dev/null +++ b/tests/unittests/kea_stats_test.py @@ -0,0 +1,47 @@ +import pytest +from nav.kea_stats import KeaQuery, KeaResponse +import requests + +def custom_post_response(func): + """ + Replace the content of the response from any call to requests.post() + with the content of func().encode("utf8") + """ + def new_post(url, *args, **kwargs): + response = requests.Response() + response._content = func().encode("utf8") + response.encoding = "utf8" + response.status_code = 400 + response.reason = "OK" + response.headers = kwargs.get("headers", {}) + response.cookies = kwargs.get("cookies", {}) + response.url = url + response.close = lambda: True + return response + + def new_post_method(self, url, *args, **kwargs): + return new_post(url, *args, **kwargs) + + def replace_post(monkeypatch): + monkeypatch.setattr(requests, 'post', new_post) + monkeypatch.setattr(requests.Session, 'post', new_post_method) # Not sure this works? + + return replace_post + +@pytest.fixture +@custom_post_response +def simple_response(): + return '[{"result": 0, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}]' + +@pytest.fixture +@custom_post_response +def lackluster_response(): + return '[{"result": "a"}]' + +@pytest.fixture +@custom_post_response +def large_response(): + return ''' + + ''' + From 85823db20d2ac14b1850b42125b6be56f21996d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 25 Jun 2024 15:20:16 +0200 Subject: [PATCH 02/77] Add dataclasses for Kea subnets and Kea DHCP config files --- python/nav/kea_stats.py | 273 +++++++++++++++++++++++++----- tests/unittests/kea_stats_test.py | 205 +++++++++++++++++++++- 2 files changed, 433 insertions(+), 45 deletions(-) diff --git a/python/nav/kea_stats.py b/python/nav/kea_stats.py index 34e971a7dd..2d53c199fb 100644 --- a/python/nav/kea_stats.py +++ b/python/nav/kea_stats.py @@ -2,21 +2,33 @@ Functions for querying the Kea Control Agent for statistics from Kea DHCP servers. -Stork (https://gitlab.isc.org/isc-projects/stork) is used as a guiding -implementation for interacting with the Kea Control Agent. See also the Kea -Control Agent documentation -(https://kea.readthedocs.io/en/kea-2.6.0/arm/agent.html). + RESTful-queries IPC +nav <---------------> Kea Control Agent <=====> Kea DHCP4 server / Kea DHCP6 server + (json) + +No additional hook libraries are assumed to be included with the Kea Control +Agent that is queried, meaning this module will be able to gather DHCP +statistics from any Kea Control Agent. + +* Stork (https://gitlab.isc.org/isc-projects/stork) is used as a guiding +implementation for interacting with the Kea Control Agent. +* See also the Kea Control Agent documentation. This script assumes Kea versions +>= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). """ from dataclasses import dataclass, asdict -from enum import Enum +from enum import IntEnum import logging import requests +import json from requests.exceptions import JSONDecodeError, HTTPError -from typing import Union +from typing import Union, Optional +from IPy import IP logger = logging.getLogger(__name__) -class KeaStatus(Enum): + +class KeaStatus(IntEnum): + """Status of a REST response.""" # Successful operation. SUCCESS = 0 # General failure. @@ -28,51 +40,234 @@ class KeaStatus(Enum): # Unsuccessful operation due to a conflict between the command arguments and the server state. CONFLICT = 4 + @dataclass class KeaResponse: """ - Class for defining a REST response on a REST query sent to a kea-ctrl-agent - process. + Class representing a REST response on a REST query sent to a Kea Control + Agent. """ result: int text: str arguments: dict[str: Union[str, int]] service: str + @property + def success(self): + return self.result == KeaStatus.SUCCESS + + @dataclass class KeaQuery: - """ - Class for defining a REST query to be sent to a kea-ctrl-agent - process. - """ + """Class representing a REST query to be sent to a Kea Control Agent.""" command: str arguments: dict[str: Union[str, int]] - services: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. + service: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. + + +@dataclass +class KeaDhcpSubnet: + """Class representing information about a subnet managed by a Kea DHCP server.""" + id: int # either specified in the server config or assigned automatically by the dhcp server + prefix: IP # e.g. 192.0.2.1/24 + pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] + + @classmethod + def from_json(cls, subnetjson: dict): + """ + Initialize and return a Subnet instance based on json + + :param json: python dictionary that is structured the same way as the + json object representing a subnet in the Kea DHCP config file. + Example: + { + "id": 0 + "subnet": "192.0.2.0/24", + "pools": [ + { + "pool": "192.0.2.1 - 192.0.2.100" + }, + { + "pool": "192.0.2.128/26" + } + ] + } + """ + if "id" not in subnetjson: + raise ValueError("Expected subnetjson['id'] to exist") + id = subnetjson["id"] + + if "subnet" not in subnetjson: + raise ValueError("Expected subnetjson['subnet'] to exist") + prefix = IP(subnetjson["subnet"]), + + pools = [] + for obj in subnetjson.get("pools", []): + pool = obj["pool"] + if "-" in pool: # TODO: Error checking? + # pool == "x.x.x.x - y.y.y.y" + start, end = (IP(ip) for ip in pool.split("-")) + else: + # pool == "x.x.x.x/nn" + pool = IP(pool) + start, end = pool[0], pool[-1] + pools.append((start, end)) + + return cls( + id=id, + prefix=prefix, + pools=pools, + ) + + +@dataclass +class KeaDhcpConfig: + """ + Class representing information found in the configuration of a Kea DHCP + server. Most importantly, this class contains: + * A list of the shared networks managed by the DHCP server + * A shared network furthermore contain a list of subnets + assigned to that network + * The IP version of the DCHP server + """ + _config_hash: Optional[str] # Used to check if there's a new config on the Kea DHCP server + ip_version: int + subnets: list[KeaDhcpSubnet] - def send(self, address) -> list[KeaResponse]: + + @classmethod + def from_json(cls, configjson: dict, hash: Optional[str] = None): """ - Send this query to a kea-ctrl-agent located at address + Initialize and return a Config instance based on json + + :param json: a dictionary that is structured the same way as a + Kea DHCP configuration. + Example: + { + "Dhcp4": { + "subnet4": [{ + "id": 1, + "subnet": "192.0.2.0/24", + "pools": [ + { + "pool": "192.0.2.1 - 192.0.2.200", + }, + ], + }] + } + } + + :param hash: hash of the Kea DHCP config file as returned by a + `config-hash-get` query on the kea-ctrl-agent REST server. """ - logger.debug("KeaQuery.send: sending request to %s with query %r", address, self) - try: - r = requests.post(address, data=asdict(self)) - except HTTPError as err: - logger.error("KeaQuery.send: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) - raise err - - try: - json = r.json() - except JSONDecodeError as err: - logger.error("KeaQuery.send: expected json from %s, got %s", address, r.text) - raise err - - responses = [] - for obj in json: - response = KeaResponse( - obj.get("result", KeaStatus.ERROR), - obj.get("text", ""), - obj.get("arguments", {}), - obj.get("service", ""), - ) - responses.append(response) - return responses + if len(configjson) > 1: + raise ValueError("Did not expect len(configjson) > 1") + + ip_version, json = configjson.popitem() + if ip_version == "Dhcp4": + ip_version = 4 + elif ip_version == "Dhcp6": + ip_version == 6 + else: + raise ValueError(f"Unsupported DHCP IP version '{ip_version}'") + + subnets = [] + for obj in json.get(f"subnet{ip_version}", []): + subnet = KeaDhcpSubnet.from_json(obj) + subnets.append(subnet) + + return cls( + _config_hash=hash, + ip_version=ip_version, + subnets=subnets, + ) + + +def send_query(query: KeaQuery, address: str, port: int, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: + """ + Internal function. + Send `query` to a Kea Control Agent listening to `port` + on IP address `address`, using either http or https + + :param session: optional session to be used when sending the query. Assumed + to not be closed. Session is not closed after the query, so that the session + can be used for persistent connections among differend send_query calls. + """ + scheme = "https" if https else "http" + location = f"{scheme}://{address}:{port}/" + logger.debug("send_query: sending request to %s with query %r", location, query) + try: + if session is None: + r = requests.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) + else: + r = session.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) + except HTTPError as err: + logger.error("send_query: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) + raise err + + try: + responsejson = r.json() + except JSONDecodeError as err: + logger.error("send_query: expected json from %s, got %s", address, r.text) + raise err + + if isinstance(responsejson, dict): + logger.error("send_query: expected a json list of objects from %s, got %r", address, responsejson) + raise ValueError(f"bad response from {address}: {responsejson!r}") + + responses = [] + for obj in responsejson: + response = KeaResponse( + obj.get("result", KeaStatus.ERROR), + obj.get("text", ""), + obj.get("arguments", {}), + obj.get("service", ""), + ) + responses.append(response) + return responses + + +def get_dhcp_config(address: str, port: int, https: bool = True, ip_version: int = 4) -> KeaDhcpConfig: + """ Fetch the config of a Kea DHCP server that manages addresses of IP + version `ip_version` from a Kea Control Agent listening to `port` on + `address`. + + :param address: the IP address or DNS addressable hostname of the Kea + Control Agent. + :param port: the port that the Kea Control Agent listens to. + :param https: whether or not to use https. If not using https, http is used. + :param ip_version: the IP version of the Kea DHCP server + """ + query = KeaQuery( + command="config-get", + service=[f"dhcp{ip_version}"], + arguments={}, + ) + responses = send_query(query, address, port, https) + if len(responses) != 1: + raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception + + response = responses[0] + if not response.success: + raise Exception("Did not receive config file from DHCP server") + + return KeaDhcpConfig.from_json(responses[0].arguments) + +def get_dhcp_statistics(address: str, port: int, https: bool = True, ip_version: int = 4) -> KeaDhcpSubnet: + query = KeaQuery( + command="statistic-get", + service=[f"dhcp{ip_version}"], + arguments={ + "name": f"subnet[1].assigned-addresses", + }, + ) + + with requests.Session() as s: + responses = send_query(query, address, port, https, session=s) + if len(responses) != 1: + raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception + + response = responses[0] + if not response.success: + raise Exception("Did not receive statistics from DHCP server") + return response diff --git a/tests/unittests/kea_stats_test.py b/tests/unittests/kea_stats_test.py index decc7e883a..351c090f4c 100644 --- a/tests/unittests/kea_stats_test.py +++ b/tests/unittests/kea_stats_test.py @@ -1,6 +1,99 @@ +from nav.kea_stats import * import pytest -from nav.kea_stats import KeaQuery, KeaResponse import requests +from IPy import IP +import json +from requests.exceptions import JSONDecodeError + +DHCP4_CONFIG = ''' + { + "Dhcp4": { + "subnet4": [{ + "4o6-interface": "eth1", + "4o6-interface-id": "ethx", + "4o6-subnet": "2001:db8:1:1::/64", + "allocator": "iterative", + "authoritative": false, + "boot-file-name": "/tmp/boot", + "client-class": "foobar", + "ddns-generated-prefix": "myhost", + "ddns-override-client-update": true, + "ddns-override-no-update": true, + "ddns-qualifying-suffix": "example.org", + "ddns-replace-client-name": "never", + "ddns-send-updates": true, + "ddns-update-on-renew": true, + "ddns-use-conflict-resolution": true, + "hostname-char-replacement": "x", + "hostname-char-set": "[^A-Za-z0-9.-]", + "id": 1, + "interface": "eth0", + "match-client-id": true, + "next-server": "0.0.0.0", + "store-extended-info": true, + "option-data": [ + { + "always-send": true, + "code": 3, + "csv-format": true, + "data": "192.0.3.1", + "name": "routers", + "space": "dhcp4" + } + ], + "pools": [ + { + "client-class": "phones_server1", + "option-data": [], + "pool": "192.1.0.1 - 192.1.0.200", + "pool-id": 7, + "require-client-classes": [ "late" ] + }, + { + "client-class": "phones_server2", + "option-data": [], + "pool": "192.3.0.1 - 192.3.0.200", + "require-client-classes": [] + } + ], + "rebind-timer": 40, + "relay": { + "ip-addresses": [ + "192.168.56.1" + ] + }, + "renew-timer": 30, + "reservations-global": true, + "reservations-in-subnet": true, + "reservations-out-of-pool": true, + "calculate-tee-times": true, + "t1-percent": 0.5, + "t2-percent": 0.75, + "cache-threshold": 0.25, + "cache-max-age": 1000, + "reservations": [ + { + "circuit-id": "01:11:22:33:44:55:66", + "ip-address": "192.0.2.204", + "hostname": "foo.example.org", + "option-data": [ + { + "name": "vivso-suboptions", + "data": "4491" + } + ] + } + ], + "require-client-classes": [ "late" ], + "server-hostname": "myhost.example.org", + "subnet": "192.0.0.0/8", + "valid-lifetime": 6000, + "min-valid-lifetime": 4000, + "max-valid-lifetime": 8000 + }] + } + } + ''' def custom_post_response(func): """ @@ -30,18 +123,118 @@ def replace_post(monkeypatch): @pytest.fixture @custom_post_response -def simple_response(): - return '[{"result": 0, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}]' +def success_responses(): + return '''[ + {"result": 0, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 0, "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 0, "service": "d"}, + {"result": 0} + ]''' + +@pytest.fixture +@custom_post_response +def error_responses(): + return '''[ + {"result": 1, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 2, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 3, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 4, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"text": "b", "arguments": {"arg1": "val1"}, "service": "d"} + ]''' @pytest.fixture @custom_post_response -def lackluster_response(): - return '[{"result": "a"}]' +def invalid_json_responses(): + return '''[ + {"result": 1, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 2, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 3, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"result": 4, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, + {"text": "b", "arguments": {"arg1": "val1"}, "service": "d" + ]''' @pytest.fixture @custom_post_response -def large_response(): +def large_responses(): return ''' ''' +def test_success_responses_does_succeed(success_responses): + query = KeaQuery("command", {}, []) + responses = send_query(query, "example.org") + assert len(responses) == 4 + for response in responses: + assert response.success + +def test_error_responses_does_not_succeed(error_responses): + query = KeaQuery("command", {}, []) + responses = send_query(query, "example.org") + assert len(responses) == 5 + for response in responses: + assert not response.success + +def test_invalid_json_responses_raises_jsonerror(invalid_json_responses): + query = KeaQuery("command", {}, []) + with pytest.raises(JSONDecodeError): + responses = send_query(query, "example.org") + +def test_correct_subnet_from_json(dhcp4_config): + j = json.loads(dhcp4_config) + subnet = Subnet.from_json(j["Dhcp4"]["subnet4"][0]) + assert subnet.id == 1 + assert subnet.prefix == IP("192.0.0.0/8") + assert len(subnet.pools) == 2 + assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) + assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) + +def test_correct_config_from_json(dhcp4_config): + j = json.loads(dhcp4_config) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 1 + subnet = config.subnets[0] + assert subnet.id == 1 + assert subnet.prefix == IP("192.0.0.0/8") + assert len(subnet.pools) == 2 + assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) + assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) + assert config.ip_version == 4 + +@pytest.fixture +@custom_post_response +def dhcp4_config_response(): + return f''' + {{ + "result": 0, + "arguments": {{ + {DHCP4_CONFIG} + }} + }} + ''' + +def test_get_dhcp_config(dhcp4_config_response): + config = get_dhcp_config("example.org", ip_version=4) + assert len(config.subnets) == 1 + subnet = config.subnets[0] + assert subnet.id == 1 + assert subnet.prefix == IP("192.0.0.0/8") + assert len(subnet.pools) == 2 + assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) + assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) + assert config.ip_version == 4 + +@pytest.fixture +@custom_post_response +def dhcp4_config_response_result_is_1(): + return f''' + {{ + "result": 1, + "arguments": {{ + {DHCP4_CONFIG} + }} + }} + ''' + +def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): + with pytest.raises(Exception): # TODO: Change + get_dhcp_config("example-org", ip_version=4) From e6ef7adabba24960bc245017a2f6f8c0b93b34f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 25 Jun 2024 16:04:20 +0200 Subject: [PATCH 03/77] Rename kea_stats.py to dhcp/kea_dhcp_data.py --- .../{kea_stats.py => dhcp/kea_dhcp_data.py} | 55 ++++++++++++------- .../kea_dhcp_data_test.py} | 6 +- 2 files changed, 39 insertions(+), 22 deletions(-) rename python/nav/{kea_stats.py => dhcp/kea_dhcp_data.py} (87%) rename tests/unittests/{kea_stats_test.py => dhcp/kea_dhcp_data_test.py} (98%) diff --git a/python/nav/kea_stats.py b/python/nav/dhcp/kea_dhcp_data.py similarity index 87% rename from python/nav/kea_stats.py rename to python/nav/dhcp/kea_dhcp_data.py index 2d53c199fb..197e399d8e 100644 --- a/python/nav/kea_stats.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -73,7 +73,7 @@ class KeaDhcpSubnet: pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] @classmethod - def from_json(cls, subnetjson: dict): + def from_json(cls, subnet_json: dict): """ Initialize and return a Subnet instance based on json @@ -93,16 +93,16 @@ def from_json(cls, subnetjson: dict): ] } """ - if "id" not in subnetjson: + if "id" not in subnet_json: raise ValueError("Expected subnetjson['id'] to exist") - id = subnetjson["id"] + id = subnet_json["id"] - if "subnet" not in subnetjson: + if "subnet" not in subnet_json: raise ValueError("Expected subnetjson['subnet'] to exist") - prefix = IP(subnetjson["subnet"]), + prefix = IP(subnet_json["subnet"]), pools = [] - for obj in subnetjson.get("pools", []): + for obj in subnet_json.get("pools", []): pool = obj["pool"] if "-" in pool: # TODO: Error checking? # pool == "x.x.x.x - y.y.y.y" @@ -119,9 +119,8 @@ def from_json(cls, subnetjson: dict): pools=pools, ) - @dataclass -class KeaDhcpConfig: +class KeaDhcpData: """ Class representing information found in the configuration of a Kea DHCP server. Most importantly, this class contains: @@ -134,11 +133,26 @@ class KeaDhcpConfig: ip_version: int subnets: list[KeaDhcpSubnet] + rest_address: str + rest_port: int + rest_https: bool + + @property + def rest_location(self): + scheme = "https" if self.rest_https else "http" + return f"{scheme}://{self.rest_address}:{self.rest_port}/" @classmethod - def from_json(cls, configjson: dict, hash: Optional[str] = None): + def from_json( + cls, + config_json: dict, + config_hash: Optional[str] = None, + rest_address: str, + rest_post: int, + rest_https: bool = True + ): """ - Initialize and return a Config instance based on json + Initialize and return a KeaDhcpData instance based on json :param json: a dictionary that is structured the same way as a Kea DHCP configuration. @@ -160,10 +174,10 @@ def from_json(cls, configjson: dict, hash: Optional[str] = None): :param hash: hash of the Kea DHCP config file as returned by a `config-hash-get` query on the kea-ctrl-agent REST server. """ - if len(configjson) > 1: + if len(config_json) > 1: raise ValueError("Did not expect len(configjson) > 1") - ip_version, json = configjson.popitem() + ip_version, json = config_json.popitem() if ip_version == "Dhcp4": ip_version = 4 elif ip_version == "Dhcp6": @@ -177,9 +191,12 @@ def from_json(cls, configjson: dict, hash: Optional[str] = None): subnets.append(subnet) return cls( - _config_hash=hash, + _config_hash=config_hash, ip_version=ip_version, subnets=subnets, + rest_address=rest_address, + rest_post=rest_post, + rest_https=rest_https, ) @@ -206,17 +223,17 @@ def send_query(query: KeaQuery, address: str, port: int, https: bool = True, ses raise err try: - responsejson = r.json() + response_json = r.json() except JSONDecodeError as err: logger.error("send_query: expected json from %s, got %s", address, r.text) raise err - if isinstance(responsejson, dict): - logger.error("send_query: expected a json list of objects from %s, got %r", address, responsejson) - raise ValueError(f"bad response from {address}: {responsejson!r}") + if isinstance(response_json, dict): + logger.error("send_query: expected a json list of objects from %s, got %r", address, response_json) + raise ValueError(f"bad response from {address}: {response_json!r}") responses = [] - for obj in responsejson: + for obj in response_json: response = KeaResponse( obj.get("result", KeaStatus.ERROR), obj.get("text", ""), @@ -227,7 +244,7 @@ def send_query(query: KeaQuery, address: str, port: int, https: bool = True, ses return responses -def get_dhcp_config(address: str, port: int, https: bool = True, ip_version: int = 4) -> KeaDhcpConfig: +def get_dhcp_server(address: str, port: int, https: bool = True, ip_version: int = 4) -> KeaDhcpConfig: """ Fetch the config of a Kea DHCP server that manages addresses of IP version `ip_version` from a Kea Control Agent listening to `port` on `address`. diff --git a/tests/unittests/kea_stats_test.py b/tests/unittests/dhcp/kea_dhcp_data_test.py similarity index 98% rename from tests/unittests/kea_stats_test.py rename to tests/unittests/dhcp/kea_dhcp_data_test.py index 351c090f4c..f16da6155f 100644 --- a/tests/unittests/kea_stats_test.py +++ b/tests/unittests/dhcp/kea_dhcp_data_test.py @@ -1,4 +1,4 @@ -from nav.kea_stats import * +from nav.dhcp.kea_dhcp_data import * import pytest import requests from IPy import IP @@ -213,7 +213,7 @@ def dhcp4_config_response(): ''' def test_get_dhcp_config(dhcp4_config_response): - config = get_dhcp_config("example.org", ip_version=4) + config = get_dhcp_server("example.org", ip_version=4) assert len(config.subnets) == 1 subnet = config.subnets[0] assert subnet.id == 1 @@ -237,4 +237,4 @@ def dhcp4_config_response_result_is_1(): def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): with pytest.raises(Exception): # TODO: Change - get_dhcp_config("example-org", ip_version=4) + get_dhcp_server("example-org", ip_version=4) From 08639a3185f21208a525aabafbfbab7a445f7a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 25 Jun 2024 17:43:43 +0200 Subject: [PATCH 04/77] Add superclass for all classes that fetch dhcp metrics Why: The main metrics one wishes to obtain from a dhcp server of a particular type (Kea DHCP, ISC DHCP, udhcpd, etc.) are the same accross the board, and thus the methods that process these metrics (e.g. sending them to a graphite server, creating a canonical graphite path for a specific type of metric, etc) are better off being defined once in a superclass. --- python/nav/dhcp/dhcp_data.py | 31 +++++++++++++++++++++++++++++++ python/nav/dhcp/kea_dhcp_data.py | 3 +-- 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 python/nav/dhcp/dhcp_data.py diff --git a/python/nav/dhcp/dhcp_data.py b/python/nav/dhcp/dhcp_data.py new file mode 100644 index 0000000000..0e82b2a614 --- /dev/null +++ b/python/nav/dhcp/dhcp_data.py @@ -0,0 +1,31 @@ +from typing import Iterator +from dataclasses import dataclass +from nav.metrics import carbon + +@dataclass +class DhcpMetric: + timestamp: int + vlan: int + key: str + value: int + +class DhcpMetricSource: + """ + Superclass for all classes that wish to collect metrics from a + specific line of DHCP servers and import the metrics into NAV's + graphite server. Subclasses need to implement `fetch_metrics`. + """ + graphite_prefix: str + + def __init__(self, graphite_prefix="nav.dhcp"): + self.graphite_prefix = graphite_prefix + + def fetch_metrics(self) -> Iterator[DhcpMetric]: + raise NotImplementedError + def fetch_metrics_to_graphite(self, host, port): + graphite_metrics = [] + for metric in self.fetch_metrics(): + graphite_path = f"{self.graphite_prefix}.vlan-{metric.vlan}.{metric.key}" + datapoint = (metric.timestamp, metric.value) + graphite_metrics.append((graphite_path, datapoint)) + carbon.send_metrics_to(graphite_metrics, host, port) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 197e399d8e..35029bb43c 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -119,8 +119,7 @@ def from_json(cls, subnet_json: dict): pools=pools, ) -@dataclass -class KeaDhcpData: +class KeaDhcpData(DhcpData): """ Class representing information found in the configuration of a Kea DHCP server. Most importantly, this class contains: From a0edbc0380e213c63e5f5f8138a134550cbc49ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 25 Jun 2024 18:04:54 +0200 Subject: [PATCH 05/77] Sort import statements --- python/nav/dhcp/dhcp_data.py | 5 ++++- python/nav/dhcp/kea_dhcp_data.py | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/python/nav/dhcp/dhcp_data.py b/python/nav/dhcp/dhcp_data.py index 0e82b2a614..34cbce53ed 100644 --- a/python/nav/dhcp/dhcp_data.py +++ b/python/nav/dhcp/dhcp_data.py @@ -1,6 +1,6 @@ -from typing import Iterator from dataclasses import dataclass from nav.metrics import carbon +from typing import Iterator @dataclass class DhcpMetric: @@ -21,6 +21,9 @@ def __init__(self, graphite_prefix="nav.dhcp"): self.graphite_prefix = graphite_prefix def fetch_metrics(self) -> Iterator[DhcpMetric]: + """ + Fetch + """ raise NotImplementedError def fetch_metrics_to_graphite(self, host, port): graphite_metrics = [] diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 35029bb43c..647eb428f4 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -15,14 +15,15 @@ * See also the Kea Control Agent documentation. This script assumes Kea versions >= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). """ -from dataclasses import dataclass, asdict -from enum import IntEnum +import json import logging import requests -import json +from dataclasses import dataclass, asdict +from .dhcp_data import DhcpMetricSource +from enum import IntEnum +from IPy import IP from requests.exceptions import JSONDecodeError, HTTPError from typing import Union, Optional -from IPy import IP logger = logging.getLogger(__name__) From 4f47a3048aa405c25ac9af85a8f830c3d01a0758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 25 Jun 2024 20:07:42 +0200 Subject: [PATCH 06/77] Add comments and more precise typing to make intentions of DhcpMetricSource clearer --- python/nav/dhcp/dhcp_data.py | 39 ++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/python/nav/dhcp/dhcp_data.py b/python/nav/dhcp/dhcp_data.py index 34cbce53ed..54b8475d7e 100644 --- a/python/nav/dhcp/dhcp_data.py +++ b/python/nav/dhcp/dhcp_data.py @@ -1,12 +1,22 @@ from dataclasses import dataclass +from enum import Enum from nav.metrics import carbon from typing import Iterator +class DhcpMetricKey(Enum): + MAX = "total addresses" + CUR = "assigned addresses" + TOUCH = "touched addresses" + FREE = "free addresses" + + def __str__(self): + return self.name.lower() # For use in graphite path + @dataclass class DhcpMetric: timestamp: int - vlan: int - key: str + vlan: int # the vlan this metric tracks + key: DhcpMetricKey value: int class DhcpMetricSource: @@ -22,13 +32,34 @@ def __init__(self, graphite_prefix="nav.dhcp"): def fetch_metrics(self) -> Iterator[DhcpMetric]: """ - Fetch + Fetch total addresses, assigned addresses, touched addresses, + and free adddresses for each vlan of the DHCP server. + + None of the DHCP server packages that has had a + DhcpMetricSource class definition so far has any way to + explicitly define which subnet or pool belongs to which + vlan. The way we figure out which subnet or pool belongs to + which vlan, differs between the DHCP server packages; usually + it is possible to give each subnet or pool or group of + subnets/pools a name, ID, or tag. The convention is that this + name, ID or tag is the vlan-number of that specific subnet or + pool or group of subnets/pools. + + Each subclass of DhcpMetricSource should document how it finds + out what vlan a subnet/pool belongs to. It should be clear + whether or not it relies on any specific conventions that the + administrator of a DHCP server must follow. + + TODO: document this properly. (do we need to specify the + rationale for grouping metrics by vlan and not + e.g. subnet-prefixes, etc. or is this clear to all users of + nav?) """ raise NotImplementedError def fetch_metrics_to_graphite(self, host, port): graphite_metrics = [] for metric in self.fetch_metrics(): - graphite_path = f"{self.graphite_prefix}.vlan-{metric.vlan}.{metric.key}" + graphite_path = f"{self.graphite_prefix}.vlan{metric.vlan}.{metric.key}" datapoint = (metric.timestamp, metric.value) graphite_metrics.append((graphite_path, datapoint)) carbon.send_metrics_to(graphite_metrics, host, port) From 879b6baae7cfd023940a5bfbecc6df0564f798a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 25 Jun 2024 20:09:22 +0200 Subject: [PATCH 07/77] Implement KeaDhcpMetricSource as subclass of DhcpMetricSource KeaDhcpMetricSource is an implementation of DhcpMetricSource which collects the four metrics defined in the DhcpMetricKeys enum (number of total, used, free and touched addresses) for each vlan of a Kea DHCP server. --- python/nav/dhcp/kea_dhcp_data.py | 200 ++++++++++++++++--------------- 1 file changed, 104 insertions(+), 96 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 647eb428f4..663be44307 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -19,7 +19,7 @@ import logging import requests from dataclasses import dataclass, asdict -from .dhcp_data import DhcpMetricSource +from .dhcp_data import DhcpMetricSource, DhcpMetric from enum import IntEnum from IPy import IP from requests.exceptions import JSONDecodeError, HTTPError @@ -65,6 +65,49 @@ class KeaQuery: arguments: dict[str: Union[str, int]] service: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. +def send_query(query: KeaQuery, address: str, port: int, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: + """ + Internal function. + Send `query` to a Kea Control Agent listening to `port` + on IP address `address`, using either http or https + + :param session: optional session to be used when sending the query. Assumed + to not be closed. Session is not closed after the query, so that the session + can be used for persistent connections among differend send_query calls. + """ + scheme = "https" if https else "http" + location = f"{scheme}://{address}:{port}/" + logger.debug("send_query: sending request to %s with query %r", location, query) + try: + if session is None: + r = requests.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) + else: + r = session.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) + except HTTPError as err: + logger.error("send_query: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) + raise err + + try: + response_json = r.json() + except JSONDecodeError as err: + logger.error("send_query: expected json from %s, got %s", address, r.text) + raise err + + if isinstance(response_json, dict): + logger.error("send_query: expected a json list of objects from %s, got %r", address, response_json) + raise ValueError(f"bad response from {address}: {response_json!r}") + + responses = [] + for obj in response_json: + response = KeaResponse( + obj.get("result", KeaStatus.ERROR), + obj.get("text", ""), + obj.get("arguments", {}), + obj.get("service", ""), + ) + responses.append(response) + return responses + @dataclass class KeaDhcpSubnet: @@ -120,7 +163,8 @@ def from_json(cls, subnet_json: dict): pools=pools, ) -class KeaDhcpData(DhcpData): +@dataclass +class KeaDhcpConfig: """ Class representing information found in the configuration of a Kea DHCP server. Most importantly, this class contains: @@ -133,23 +177,11 @@ class KeaDhcpData(DhcpData): ip_version: int subnets: list[KeaDhcpSubnet] - rest_address: str - rest_port: int - rest_https: bool - - @property - def rest_location(self): - scheme = "https" if self.rest_https else "http" - return f"{scheme}://{self.rest_address}:{self.rest_port}/" - @classmethod def from_json( cls, config_json: dict, config_hash: Optional[str] = None, - rest_address: str, - rest_post: int, - rest_https: bool = True ): """ Initialize and return a KeaDhcpData instance based on json @@ -194,97 +226,73 @@ def from_json( _config_hash=config_hash, ip_version=ip_version, subnets=subnets, - rest_address=rest_address, - rest_post=rest_post, - rest_https=rest_https, ) -def send_query(query: KeaQuery, address: str, port: int, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: - """ - Internal function. - Send `query` to a Kea Control Agent listening to `port` - on IP address `address`, using either http or https - - :param session: optional session to be used when sending the query. Assumed - to not be closed. Session is not closed after the query, so that the session - can be used for persistent connections among differend send_query calls. - """ - scheme = "https" if https else "http" - location = f"{scheme}://{address}:{port}/" - logger.debug("send_query: sending request to %s with query %r", location, query) - try: - if session is None: - r = requests.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) - else: - r = session.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) - except HTTPError as err: - logger.error("send_query: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) - raise err +class KeaDhcpMetricSource(DhcpMetricSource): + rest_address: str # IP address of the Kea Control Agent server + rest_port: int # Port of the Kea Control Agent server + rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. - try: - response_json = r.json() - except JSONDecodeError as err: - logger.error("send_query: expected json from %s, got %s", address, r.text) - raise err + ip_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server + kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server that is used as a data/metric source - if isinstance(response_json, dict): - logger.error("send_query: expected a json list of objects from %s, got %r", address, response_json) - raise ValueError(f"bad response from {address}: {response_json!r}") + def __init__(self, address: str, port: int, https: bool = True, ip_version: int = 4, *args, **kwargs): + super(*args, **kwargs) + self.rest_address = address + self.rest_port = port + self.rest_https = https + self.ip_version = ip_version + self.kea_dhcp_config = None - responses = [] - for obj in response_json: - response = KeaResponse( - obj.get("result", KeaStatus.ERROR), - obj.get("text", ""), - obj.get("arguments", {}), - obj.get("service", ""), + def fetch_dhcp_config(self) -> KeaDhcpConfig: + """ + Fetch the config of the Kea DHCP server that manages addresses of IP + version `self.ip_version` from the Kea Control Agent listening to + `self.rest_port` on `self.rest_address`. + """ + query = KeaQuery( + command="config-get", + service=[f"dhcp{ip_version}"], + arguments={}, ) - responses.append(response) - return responses - - -def get_dhcp_server(address: str, port: int, https: bool = True, ip_version: int = 4) -> KeaDhcpConfig: - """ Fetch the config of a Kea DHCP server that manages addresses of IP - version `ip_version` from a Kea Control Agent listening to `port` on - `address`. - - :param address: the IP address or DNS addressable hostname of the Kea - Control Agent. - :param port: the port that the Kea Control Agent listens to. - :param https: whether or not to use https. If not using https, http is used. - :param ip_version: the IP version of the Kea DHCP server - """ - query = KeaQuery( - command="config-get", - service=[f"dhcp{ip_version}"], - arguments={}, - ) - responses = send_query(query, address, port, https) - if len(responses) != 1: - raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception - - response = responses[0] - if not response.success: - raise Exception("Did not receive config file from DHCP server") - - return KeaDhcpConfig.from_json(responses[0].arguments) - -def get_dhcp_statistics(address: str, port: int, https: bool = True, ip_version: int = 4) -> KeaDhcpSubnet: - query = KeaQuery( - command="statistic-get", - service=[f"dhcp{ip_version}"], - arguments={ - "name": f"subnet[1].assigned-addresses", - }, - ) - - with requests.Session() as s: - responses = send_query(query, address, port, https, session=s) + responses = send_query(query, self.rest_address, self.rest_port, self.rest_https) if len(responses) != 1: raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception response = responses[0] if not response.success: - raise Exception("Did not receive statistics from DHCP server") - return response + raise Exception("Did not receive config file from DHCP server") + + self.kea_dhcp_config = KeaDhcpConfig.from_json(responses[0].arguments) + return kea_dhcp_config + + def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: int = 4) -> list[DhcpMetric]: + """ + Implementation of the superclass method for fetching + standardised dhcp metrics; this method is what nav uses to + feed data into the graphite server. + """ + + metrics = [] + with requests.Session() as s: + for subnet in self.kea_dhcp_config.subnets: + for kea_key, dhcpmetric_key in ("total-addresses","max"), ("assigned-addresses","cur"), ("","touch"), (,"free") # dhcmetric_key is the same as the graphite metric names used in nav/contrib/scripts/isc_dhpcd_graphite/isc_dhpcd_graphite.py + query = KeaQuery( + command="statistic-get", + service=[f"dhcp{self.ip_version}"], + arguments={ + "name": f"subnet[{subnet.id}].{kea_key}", + }, + ) + + responses = send_query(query, address, port, https, session=s) + if len(responses) != 1: + raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception + + response = responses[0] + if not response.success: + raise Exception("Did not receive statistics from DHCP server") + + + return response From 48ebfcf19da1242084532134abfd65292b52d07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 26 Jun 2024 09:56:59 +0200 Subject: [PATCH 08/77] Remove "free addresses" as a Dhcp metric key temporarily Why: I'll need to check more carefully how to obtain the amount of free addresses in a subnet, it proboably must be calculated since Kea doesn't seem to supply it. --- python/nav/dhcp/dhcp_data.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/nav/dhcp/dhcp_data.py b/python/nav/dhcp/dhcp_data.py index 54b8475d7e..360470f744 100644 --- a/python/nav/dhcp/dhcp_data.py +++ b/python/nav/dhcp/dhcp_data.py @@ -4,10 +4,9 @@ from typing import Iterator class DhcpMetricKey(Enum): - MAX = "total addresses" - CUR = "assigned addresses" - TOUCH = "touched addresses" - FREE = "free addresses" + MAX = "max" # total addresses + CUR = "cur" # assigned addresses + TOUCH = "touch" # touched addresses def __str__(self): return self.name.lower() # For use in graphite path From 1f65f3fcbdbf50c6290b9824e06c7101104cd9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 26 Jun 2024 09:58:27 +0200 Subject: [PATCH 09/77] Finish metric extraction from configured subnets Need to see how to deal with shared networks that might also be defined as well. Should be exactly the same as for subnets, since a shared network really is just a uniquely named list of subnets. --- python/nav/dhcp/kea_dhcp_data.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 663be44307..80ae876df1 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -15,11 +15,13 @@ * See also the Kea Control Agent documentation. This script assumes Kea versions >= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). """ +import calendar import json import logging import requests +import time from dataclasses import dataclass, asdict -from .dhcp_data import DhcpMetricSource, DhcpMetric +from .dhcp_data import DhcpMetricSource, DhcpMetric, DhcpMetricKey from enum import IntEnum from IPy import IP from requests.exceptions import JSONDecodeError, HTTPError @@ -277,12 +279,15 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: metrics = [] with requests.Session() as s: for subnet in self.kea_dhcp_config.subnets: - for kea_key, dhcpmetric_key in ("total-addresses","max"), ("assigned-addresses","cur"), ("","touch"), (,"free") # dhcmetric_key is the same as the graphite metric names used in nav/contrib/scripts/isc_dhpcd_graphite/isc_dhpcd_graphite.py + for kea_key, dhcpmetric_key in (("total-addresses", DhcpMetricKey.MAX), + ("assigned-addresses", DhcpMetricKey.CUR), + ("declined-addresses", DhcpMetricKey.TOUCH)): # dhcmetric_key is the same as the graphite metric names used in nav/contrib/scripts/isc_dhpcd_graphite/isc_dhpcd_graphite.py + kea_statistic_name = f"subnet[{subnet.id}].{kea_key}", query = KeaQuery( command="statistic-get", service=[f"dhcp{self.ip_version}"], arguments={ - "name": f"subnet[{subnet.id}].{kea_key}", + "name": kea_statistic_name, }, ) @@ -294,5 +299,9 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: if not response.success: raise Exception("Did not receive statistics from DHCP server") + datapoints = response["arguments"].get(kea_statistic_name, []) + for value, timestamp in datapoints: + epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! + metrics.append(DhcpMetric(epochseconds, subnet.id, dhcpmetric_key, value)) - return response + return metrics From 3c39713cd55eaa25b5e820f36b6a36c9492bdbfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 26 Jun 2024 16:18:28 +0200 Subject: [PATCH 10/77] Rename fetch_dhcp_config() to set_and_fetch_dhcp_config() why: This is build-up for an up-coming commit that implements caching of self.kea_dhcp_config in KeaDhcpMetricSource; everytime we fetch a new kea_dhcp_config with fetch_dhcp_config(), we would like to store it as well - hence the name change of the function. The up-coming commit will then make use of Kea Control Agent's `config-hash-get` command (included in Kea versions >= 2.4.0) to check if we need to update the cached config or not whenever set_and_fetch_dhcp_config() is called. --- python/nav/dhcp/kea_dhcp_data.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 80ae876df1..e133566a0d 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -175,7 +175,7 @@ class KeaDhcpConfig: assigned to that network * The IP version of the DCHP server """ - _config_hash: Optional[str] # Used to check if there's a new config on the Kea DHCP server + config_hash: Optional[str] # Used to check if there's a new config on the Kea DHCP server ip_version: int subnets: list[KeaDhcpSubnet] @@ -247,7 +247,7 @@ def __init__(self, address: str, port: int, https: bool = True, ip_version: int self.ip_version = ip_version self.kea_dhcp_config = None - def fetch_dhcp_config(self) -> KeaDhcpConfig: + def fetch_and_set_dhcp_config(self, session=None): """ Fetch the config of the Kea DHCP server that manages addresses of IP version `self.ip_version` from the Kea Control Agent listening to @@ -258,7 +258,7 @@ def fetch_dhcp_config(self) -> KeaDhcpConfig: service=[f"dhcp{ip_version}"], arguments={}, ) - responses = send_query(query, self.rest_address, self.rest_port, self.rest_https) + responses = send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session) if len(responses) != 1: raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception @@ -266,8 +266,9 @@ def fetch_dhcp_config(self) -> KeaDhcpConfig: if not response.success: raise Exception("Did not receive config file from DHCP server") - self.kea_dhcp_config = KeaDhcpConfig.from_json(responses[0].arguments) - return kea_dhcp_config + self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) + return self.kea_dhcp_config + def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: int = 4) -> list[DhcpMetric]: """ @@ -275,14 +276,14 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: standardised dhcp metrics; this method is what nav uses to feed data into the graphite server. """ - metrics = [] with requests.Session() as s: + self.fetch_and_set_dhcp_config(s) for subnet in self.kea_dhcp_config.subnets: - for kea_key, dhcpmetric_key in (("total-addresses", DhcpMetricKey.MAX), - ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH)): # dhcmetric_key is the same as the graphite metric names used in nav/contrib/scripts/isc_dhpcd_graphite/isc_dhpcd_graphite.py - kea_statistic_name = f"subnet[{subnet.id}].{kea_key}", + for statistic_key, metric_key in (("total-addresses", DhcpMetricKey.MAX), + ("assigned-addresses", DhcpMetricKey.CUR), + ("declined-addresses", DhcpMetricKey.TOUCH)): # dhcmetric_key is the same as the graphite metric names used in nav/contrib/scripts/isc_dhpcd_graphite/isc_dhpcd_graphite.py + kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}", query = KeaQuery( command="statistic-get", service=[f"dhcp{self.ip_version}"], From fdc497dcbe48d593a945e5ce8a7cf6887c43d1f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 26 Jun 2024 17:55:16 +0200 Subject: [PATCH 11/77] Collect metrics per subnet instead of per vlan in DhcpMetricSource --- python/nav/dhcp/dhcp_data.py | 30 ++++++------------------------ python/nav/dhcp/kea_dhcp_data.py | 2 +- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/python/nav/dhcp/dhcp_data.py b/python/nav/dhcp/dhcp_data.py index 360470f744..9471b53018 100644 --- a/python/nav/dhcp/dhcp_data.py +++ b/python/nav/dhcp/dhcp_data.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from enum import Enum +from IPy import IP from nav.metrics import carbon from typing import Iterator @@ -14,7 +15,7 @@ def __str__(self): @dataclass class DhcpMetric: timestamp: int - vlan: int # the vlan this metric tracks + subnet_prefix: IP key: DhcpMetricKey value: int @@ -31,34 +32,15 @@ def __init__(self, graphite_prefix="nav.dhcp"): def fetch_metrics(self) -> Iterator[DhcpMetric]: """ - Fetch total addresses, assigned addresses, touched addresses, - and free adddresses for each vlan of the DHCP server. - - None of the DHCP server packages that has had a - DhcpMetricSource class definition so far has any way to - explicitly define which subnet or pool belongs to which - vlan. The way we figure out which subnet or pool belongs to - which vlan, differs between the DHCP server packages; usually - it is possible to give each subnet or pool or group of - subnets/pools a name, ID, or tag. The convention is that this - name, ID or tag is the vlan-number of that specific subnet or - pool or group of subnets/pools. - - Each subclass of DhcpMetricSource should document how it finds - out what vlan a subnet/pool belongs to. It should be clear - whether or not it relies on any specific conventions that the - administrator of a DHCP server must follow. - - TODO: document this properly. (do we need to specify the - rationale for grouping metrics by vlan and not - e.g. subnet-prefixes, etc. or is this clear to all users of - nav?) + Fetch DhcpMetrics having keys `MAX`, `CUR`, `TOUCH` and `FREE` + for each subnet of the DHCP server at current point of time. """ raise NotImplementedError def fetch_metrics_to_graphite(self, host, port): + fmt = str.maketrans({".": "_", "/": "_"}) # 192.0.2.0/24 --> 192_0_0_0_24 graphite_metrics = [] for metric in self.fetch_metrics(): - graphite_path = f"{self.graphite_prefix}.vlan{metric.vlan}.{metric.key}" + graphite_path = f"{self.graphite_prefix}.{str(metric.subnet_prefix).translate(fmt)}.{metric.key}" datapoint = (metric.timestamp, metric.value) graphite_metrics.append((graphite_path, datapoint)) carbon.send_metrics_to(graphite_metrics, host, port) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index e133566a0d..a4af97a4d9 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -303,6 +303,6 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: datapoints = response["arguments"].get(kea_statistic_name, []) for value, timestamp in datapoints: epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! - metrics.append(DhcpMetric(epochseconds, subnet.id, dhcpmetric_key, value)) + metrics.append(DhcpMetric(epochseconds, subnet.prefix, metric_key, value)) return metrics From 29b82e80ed3e231e5c35501f1c929ab43fbbf15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 26 Jun 2024 17:56:36 +0200 Subject: [PATCH 12/77] Check if config hash on local and on server is equal before fetching new config --- python/nav/dhcp/kea_dhcp_data.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index a4af97a4d9..9de3d701f3 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -253,6 +253,15 @@ def fetch_and_set_dhcp_config(self, session=None): version `self.ip_version` from the Kea Control Agent listening to `self.rest_port` on `self.rest_address`. """ + # Check if self.kea_dhcp_config is up to date + if not ( + self.kea_dhcp_config is None + or self.kea_dhcp_config.config_hash is None + or self.fetch_dhcp_config_hash(session=s) != self.kea_dhcp_config.config_hash + ): + return self.kea_dhcp_config + + # self.kea_dhcp_config is not up to date, fetch new query = KeaQuery( command="config-get", service=[f"dhcp{ip_version}"], @@ -269,6 +278,24 @@ def fetch_and_set_dhcp_config(self, session=None): self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) return self.kea_dhcp_config + def fetch_dhcp_config_hash(self, session=None): + query = KeaQuery( + command="config-hash-get", + service=[f"dhcp{ip_version}"], + arguments={}, + ) + responses = send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session) + if len(responses) != 1: + Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception + + response = responses[0] + if response.result == KeaStatus.UNSUPPORTED: + logger.info("Kea DHCP%d server does not support quering for the hash of its config", ip_version) + return None + elif response.success: + return response.arguments.get("hash", None) + else: + raise Exception("Did not receive hash of config file from DHCP server") def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: int = 4) -> list[DhcpMetric]: """ From 03f2b8c3780b2435a9399c9a042d1c025ac795f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 26 Jun 2024 17:59:35 +0200 Subject: [PATCH 13/77] Warn if config changed on server during fetching of metrics --- python/nav/dhcp/kea_dhcp_data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 9de3d701f3..0a47d19045 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -332,4 +332,14 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! metrics.append(DhcpMetric(epochseconds, subnet.prefix, metric_key, value)) + + used_config = self.kea_dhcp_config + self.fetch_and_set_dhcp_config(s) + if sorted(used_config.subnets) != sorted(self.kea_dhcp_config.subnets): + logger.warning( + "Subnet configuration was modified during metric fetching, " + "this may cause metric data being associated with wrong " + "subnet." + ) + return metrics From 4ce605f08fae5b4fded09284e45e7f1b307f861c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Thu, 27 Jun 2024 12:32:58 +0200 Subject: [PATCH 14/77] Rewrite tests to work with newest code --- python/nav/dhcp/kea_dhcp_data.py | 4 +- tests/unittests/dhcp/kea_dhcp_data_test.py | 170 ++++++++++++--------- 2 files changed, 97 insertions(+), 77 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 0a47d19045..bad605a0e3 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -67,7 +67,7 @@ class KeaQuery: arguments: dict[str: Union[str, int]] service: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. -def send_query(query: KeaQuery, address: str, port: int, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: +def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: """ Internal function. Send `query` to a Kea Control Agent listening to `port` @@ -145,7 +145,7 @@ def from_json(cls, subnet_json: dict): if "subnet" not in subnet_json: raise ValueError("Expected subnetjson['subnet'] to exist") - prefix = IP(subnet_json["subnet"]), + prefix = IP(subnet_json["subnet"]) pools = [] for obj in subnet_json.get("pools", []): diff --git a/tests/unittests/dhcp/kea_dhcp_data_test.py b/tests/unittests/dhcp/kea_dhcp_data_test.py index f16da6155f..fa2aa5b9b6 100644 --- a/tests/unittests/dhcp/kea_dhcp_data_test.py +++ b/tests/unittests/dhcp/kea_dhcp_data_test.py @@ -1,3 +1,4 @@ +from collections import deque from nav.dhcp.kea_dhcp_data import * import pytest import requests @@ -5,7 +6,9 @@ import json from requests.exceptions import JSONDecodeError -DHCP4_CONFIG = ''' +@pytest.fixture +def dhcp4_config(): + return ''' { "Dhcp4": { "subnet4": [{ @@ -95,14 +98,22 @@ } ''' -def custom_post_response(func): +@pytest.fixture(autouse=True) +def enqueue_post_response(monkeypatch): """ - Replace the content of the response from any call to requests.post() - with the content of func().encode("utf8") + Any test that include this fixture, gets access to a function that + can be used to append text strings to a fifo queue of post + responses that in fifo order will be returned as proper Response + objects by calls to requests.post and requests.Session().post. + + This is how we mock what would otherwise be post requests to a + server. """ - def new_post(url, *args, **kwargs): + fifo = deque() # stores the textual content of the responses we want to return on any call to requests.post + + def new_post_function(url, *args, **kwargs): response = requests.Response() - response._content = func().encode("utf8") + response._content = fifo.popleft().encode("utf8") response.encoding = "utf8" response.status_code = 400 response.reason = "OK" @@ -113,17 +124,15 @@ def new_post(url, *args, **kwargs): return response def new_post_method(self, url, *args, **kwargs): - return new_post(url, *args, **kwargs) + return new_post_function(url, *args, **kwargs) - def replace_post(monkeypatch): - monkeypatch.setattr(requests, 'post', new_post) - monkeypatch.setattr(requests.Session, 'post', new_post_method) # Not sure this works? + monkeypatch.setattr(requests, 'post', new_post_function) + monkeypatch.setattr(requests.Session, 'post', new_post_method) # Not sure this works? - return replace_post + return fifo.append @pytest.fixture -@custom_post_response -def success_responses(): +def success_response(): return '''[ {"result": 0, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, {"result": 0, "arguments": {"arg1": "val1"}, "service": "d"}, @@ -132,8 +141,7 @@ def success_responses(): ]''' @pytest.fixture -@custom_post_response -def error_responses(): +def error_response(): return '''[ {"result": 1, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, {"result": 2, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, @@ -143,8 +151,7 @@ def error_responses(): ]''' @pytest.fixture -@custom_post_response -def invalid_json_responses(): +def invalid_json_response(): return '''[ {"result": 1, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, {"result": 2, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, @@ -154,87 +161,100 @@ def invalid_json_responses(): ]''' @pytest.fixture -@custom_post_response -def large_responses(): +def large_response(): return ''' ''' -def test_success_responses_does_succeed(success_responses): - query = KeaQuery("command", {}, []) - responses = send_query(query, "example.org") +def send_dummy_query(): + return send_query( + query=KeaQuery("command", [], {}), + address="192.0.2.2", + port=80, + ) + +def test_success_responses_does_succeed(success_response, enqueue_post_response): + enqueue_post_response(success_response) + responses = send_dummy_query() assert len(responses) == 4 for response in responses: assert response.success + assert isinstance(response.text, str) + assert isinstance(response.arguments, dict) + assert isinstance(response.service, str) -def test_error_responses_does_not_succeed(error_responses): - query = KeaQuery("command", {}, []) - responses = send_query(query, "example.org") +def test_error_responses_does_not_succeed(error_response, enqueue_post_response): + enqueue_post_response(error_response) + responses = send_dummy_query() assert len(responses) == 5 for response in responses: assert not response.success + assert isinstance(response.text, str) + assert isinstance(response.arguments, dict) + assert isinstance(response.service, str) -def test_invalid_json_responses_raises_jsonerror(invalid_json_responses): - query = KeaQuery("command", {}, []) +def test_invalid_json_responses_raises_jsonerror(invalid_json_response, enqueue_post_response): + enqueue_post_response(invalid_json_response) with pytest.raises(JSONDecodeError): - responses = send_query(query, "example.org") + responses = send_dummy_query() def test_correct_subnet_from_json(dhcp4_config): j = json.loads(dhcp4_config) - subnet = Subnet.from_json(j["Dhcp4"]["subnet4"][0]) + subnet = KeaDhcpSubnet.from_json(j["Dhcp4"]["subnet4"][0]) assert subnet.id == 1 assert subnet.prefix == IP("192.0.0.0/8") assert len(subnet.pools) == 2 assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) -def test_correct_config_from_json(dhcp4_config): - j = json.loads(dhcp4_config) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 1 - subnet = config.subnets[0] - assert subnet.id == 1 - assert subnet.prefix == IP("192.0.0.0/8") - assert len(subnet.pools) == 2 - assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) - assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) - assert config.ip_version == 4 +# def test_correct_config_from_json(dhcp4_config): +# j = json.loads(dhcp4_config) +# config = KeaDhcpConfig.from_json(j) +# assert len(config.subnets) == 1 +# subnet = config.subnets[0] +# assert subnet.id == 1 +# assert subnet.prefix == IP("192.0.0.0/8") +# assert len(subnet.pools) == 2 +# assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) +# assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) +# assert config.ip_version == 4 -@pytest.fixture -@custom_post_response -def dhcp4_config_response(): - return f''' - {{ - "result": 0, - "arguments": {{ - {DHCP4_CONFIG} - }} - }} - ''' +# @pytest.fixture +# def dhcp4_config_response(dhcp4_config): +# return f''' +# {{ +# "result": 0, +# "arguments": {{ +# {dhcp4_config} +# }} +# }} +# ''' -def test_get_dhcp_config(dhcp4_config_response): - config = get_dhcp_server("example.org", ip_version=4) - assert len(config.subnets) == 1 - subnet = config.subnets[0] - assert subnet.id == 1 - assert subnet.prefix == IP("192.0.0.0/8") - assert len(subnet.pools) == 2 - assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) - assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) - assert config.ip_version == 4 +# def test_get_dhcp_config(dhcp4_config_response): +# enqueue_post_response(dhcp4_config_response) +# response = send_dummy_query() +# config = KeaDhcpConfig.from_json(config_json) +# assert len(config.subnets) == 1 +# subnet = config.subnets[0] +# assert subnet.id == 1 +# assert subnet.prefix == IP("192.0.0.0/8") +# assert len(subnet.pools) == 2 +# assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) +# assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) +# assert config.ip_version == 4 -@pytest.fixture -@custom_post_response -def dhcp4_config_response_result_is_1(): - return f''' - {{ - "result": 1, - "arguments": {{ - {DHCP4_CONFIG} - }} - }} - ''' +# @pytest.fixture +# @enqueue_post_response +# def dhcp4_config_response_result_is_1(): +# return f''' +# {{ +# "result": 1, +# "arguments": {{ +# {DHCP4_CONFIG} +# }} +# }} +# ''' -def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): - with pytest.raises(Exception): # TODO: Change - get_dhcp_server("example-org", ip_version=4) +# def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): +# with pytest.raises(Exception): # TODO: Change +# get_dhcp_server("example-org", ip_version=4) From 0ca7edc4491a52eac4b6a8dbf1e2a1aebe82bf1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 28 Jun 2024 13:15:00 +0200 Subject: [PATCH 15/77] Update kea_dhcp_data tests to work with most recent changes What: In addition to updating function names etc., I've also updated the mocking of the requests.post requests in the test script so that we can give different respsonses based on the Kea Control Agent command the kea_dhcp_data script sends to the Kea Control Agent server with its request.post calls --- tests/unittests/dhcp/kea_dhcp_data_test.py | 144 ++++++++++++++------- 1 file changed, 97 insertions(+), 47 deletions(-) diff --git a/tests/unittests/dhcp/kea_dhcp_data_test.py b/tests/unittests/dhcp/kea_dhcp_data_test.py index fa2aa5b9b6..d7e3743c06 100644 --- a/tests/unittests/dhcp/kea_dhcp_data_test.py +++ b/tests/unittests/dhcp/kea_dhcp_data_test.py @@ -109,11 +109,45 @@ def enqueue_post_response(monkeypatch): This is how we mock what would otherwise be post requests to a server. """ - fifo = deque() # stores the textual content of the responses we want to return on any call to requests.post + command_responses = {} # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K + unknown_command_response = """[ + { + "result": 2, + "text": "'{0}' command not supported." + } +]""" + + def new_post_function(url, *args, data="{}", **kwargs): + if isinstance(data, dict): + data = json.dumps(data) + elif isinstance(data, bytes): + data = data.decode("utf8") + if not isinstance(data, str): + pytest.fail(f"data argument to the mocked requests.post() is of unknown type {type(data)}") + + try: + data = json.loads(data) + command = data["command"] + except (JSONDecodeError, KeyError): + pytest.fail( + "All post requests that the Kea Control Agent receives from NAV" + "should be a JSON with a 'command' key. Instead, the mocked Kea " + f"Control Agent received {data!r}" + ) + + fifo = command_responses.get(command, deque()) + if fifo: + first = fifo[0] + if callable(first): + text = first() + else: + text = str(first) + fifo.popleft() + else: + text = unknown_command_response.format(command) - def new_post_function(url, *args, **kwargs): response = requests.Response() - response._content = fifo.popleft().encode("utf8") + response._content = text.encode("utf8") response.encoding = "utf8" response.status_code = 400 response.reason = "OK" @@ -126,10 +160,14 @@ def new_post_function(url, *args, **kwargs): def new_post_method(self, url, *args, **kwargs): return new_post_function(url, *args, **kwargs) + def add_command_response(command_name, text): + command_responses.setdefault(command_name, deque()) + command_responses[command_name].append(text) + monkeypatch.setattr(requests, 'post', new_post_function) - monkeypatch.setattr(requests.Session, 'post', new_post_method) # Not sure this works? + monkeypatch.setattr(requests.Session, 'post', new_post_method) - return fifo.append + return add_command_response @pytest.fixture def success_response(): @@ -166,16 +204,20 @@ def large_response(): ''' -def send_dummy_query(): +def send_dummy_query(command="command"): return send_query( - query=KeaQuery("command", [], {}), + query=KeaQuery(command, [], {}), address="192.0.2.2", port=80, ) +################################################################################ +# Testing the list[KeaResponse] returned by send_query() # +################################################################################ + def test_success_responses_does_succeed(success_response, enqueue_post_response): - enqueue_post_response(success_response) - responses = send_dummy_query() + enqueue_post_response("command", success_response) + responses = send_dummy_query("command") assert len(responses) == 4 for response in responses: assert response.success @@ -184,8 +226,8 @@ def test_success_responses_does_succeed(success_response, enqueue_post_response) assert isinstance(response.service, str) def test_error_responses_does_not_succeed(error_response, enqueue_post_response): - enqueue_post_response(error_response) - responses = send_dummy_query() + enqueue_post_response("command", error_response) + responses = send_dummy_query("command") assert len(responses) == 5 for response in responses: assert not response.success @@ -194,11 +236,16 @@ def test_error_responses_does_not_succeed(error_response, enqueue_post_response) assert isinstance(response.service, str) def test_invalid_json_responses_raises_jsonerror(invalid_json_response, enqueue_post_response): - enqueue_post_response(invalid_json_response) + enqueue_post_response("command", invalid_json_response) with pytest.raises(JSONDecodeError): - responses = send_dummy_query() + responses = send_dummy_query("command") + -def test_correct_subnet_from_json(dhcp4_config): +################################################################################ +# Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from json # +################################################################################ + +def test_correct_subnet_from_dhcp4_config_json(dhcp4_config): j = json.loads(dhcp4_config) subnet = KeaDhcpSubnet.from_json(j["Dhcp4"]["subnet4"][0]) assert subnet.id == 1 @@ -207,41 +254,44 @@ def test_correct_subnet_from_json(dhcp4_config): assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) -# def test_correct_config_from_json(dhcp4_config): -# j = json.loads(dhcp4_config) -# config = KeaDhcpConfig.from_json(j) -# assert len(config.subnets) == 1 -# subnet = config.subnets[0] -# assert subnet.id == 1 -# assert subnet.prefix == IP("192.0.0.0/8") -# assert len(subnet.pools) == 2 -# assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) -# assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) -# assert config.ip_version == 4 +def test_correct_config_from_dhcp4_config_json(dhcp4_config): + j = json.loads(dhcp4_config) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 1 + subnet = config.subnets[0] + assert subnet.id == 1 + assert subnet.prefix == IP("192.0.0.0/8") + assert len(subnet.pools) == 2 + assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) + assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) + assert config.ip_version == 4 + assert config.config_hash is None -# @pytest.fixture -# def dhcp4_config_response(dhcp4_config): -# return f''' -# {{ -# "result": 0, -# "arguments": {{ -# {dhcp4_config} -# }} -# }} -# ''' +@pytest.fixture +def dhcp4_config_response(dhcp4_config): + return f''' + [ + {{ + "result": 0, + "arguments": {dhcp4_config} + }} + ] + ''' + +################################################################################ +# Now we assume KeaDhcpSubnet and KeaDhcpConfig instantiation from json is # +# correct. # +# Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from server responses # +################################################################################ -# def test_get_dhcp_config(dhcp4_config_response): -# enqueue_post_response(dhcp4_config_response) -# response = send_dummy_query() -# config = KeaDhcpConfig.from_json(config_json) -# assert len(config.subnets) == 1 -# subnet = config.subnets[0] -# assert subnet.id == 1 -# assert subnet.prefix == IP("192.0.0.0/8") -# assert len(subnet.pools) == 2 -# assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) -# assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) -# assert config.ip_version == 4 +def test_fetch_and_set_dhcp_config(dhcp4_config_response, enqueue_post_response): + enqueue_post_response("config-get", dhcp4_config_response) + source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) + assert source.kea_dhcp_config is None + config = source.fetch_and_set_dhcp_config() + actual_config = KeaDhcpConfig.from_json(json.loads(dhcp4_config_response)[0]["arguments"]) + assert config == actual_config + assert source.kea_dhcp_config == actual_config # @pytest.fixture # @enqueue_post_response From c3e34ea958345d761c515719e5b871bc9404585f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 28 Jun 2024 13:18:59 +0200 Subject: [PATCH 16/77] Update variable names --- python/nav/dhcp/kea_dhcp_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index bad605a0e3..e3c22970da 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -225,7 +225,7 @@ def from_json( subnets.append(subnet) return cls( - _config_hash=config_hash, + config_hash=config_hash, ip_version=ip_version, subnets=subnets, ) @@ -264,7 +264,7 @@ def fetch_and_set_dhcp_config(self, session=None): # self.kea_dhcp_config is not up to date, fetch new query = KeaQuery( command="config-get", - service=[f"dhcp{ip_version}"], + service=[f"dhcp{self.ip_version}"], arguments={}, ) responses = send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session) @@ -281,7 +281,7 @@ def fetch_and_set_dhcp_config(self, session=None): def fetch_dhcp_config_hash(self, session=None): query = KeaQuery( command="config-hash-get", - service=[f"dhcp{ip_version}"], + service=[f"dhcp{self.ip_version}"], arguments={}, ) responses = send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session) From 85da641b0cddc5826b42ed332f47f327304fcdd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 28 Jun 2024 13:58:04 +0200 Subject: [PATCH 17/77] Move error checks done after send_query() calls to a new function For now, I'll just call this new function for unwrap(), since all of the uses of send_query() expects a list of one response and the first thing that is done is always to unwrap the singleton response list. unwrap() could be made to have generic typing, but for simplicity it uses KeaResponse instead of a generic TypeVar('T') for now. --- python/nav/dhcp/kea_dhcp_data.py | 40 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index e3c22970da..6529660445 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -24,11 +24,14 @@ from .dhcp_data import DhcpMetricSource, DhcpMetric, DhcpMetricKey from enum import IntEnum from IPy import IP +from nav.errors import GeneralException from requests.exceptions import JSONDecodeError, HTTPError from typing import Union, Optional logger = logging.getLogger(__name__) +class KeaError(GeneralException): + pass class KeaStatus(IntEnum): """Status of a REST response.""" @@ -97,7 +100,7 @@ def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = Tru if isinstance(response_json, dict): logger.error("send_query: expected a json list of objects from %s, got %r", address, response_json) - raise ValueError(f"bad response from {address}: {response_json!r}") + raise KeaError(f"bad response from {address}: {response_json!r}") responses = [] for obj in response_json: @@ -110,6 +113,14 @@ def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = Tru responses.append(response) return responses +def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: + if len(responses) != 1: + raise KeaError(f"Received invalid amount of responses") + + response = responses[0] + if require_success and not response.success: + raise KeaError("Did not receive a successful response") + return response @dataclass class KeaDhcpSubnet: @@ -267,13 +278,7 @@ def fetch_and_set_dhcp_config(self, session=None): service=[f"dhcp{self.ip_version}"], arguments={}, ) - responses = send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session) - if len(responses) != 1: - raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception - - response = responses[0] - if not response.success: - raise Exception("Did not receive config file from DHCP server") + response = unwrap(send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session)) self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) return self.kea_dhcp_config @@ -284,18 +289,18 @@ def fetch_dhcp_config_hash(self, session=None): service=[f"dhcp{self.ip_version}"], arguments={}, ) - responses = send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session) - if len(responses) != 1: - Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception + response = unwrap( + send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session), + require_success=False + ) - response = responses[0] if response.result == KeaStatus.UNSUPPORTED: logger.info("Kea DHCP%d server does not support quering for the hash of its config", ip_version) return None elif response.success: return response.arguments.get("hash", None) else: - raise Exception("Did not receive hash of config file from DHCP server") + raise KeaError("Unexpected error when querying the hash of config file from DHCP server") def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: int = 4) -> list[DhcpMetric]: """ @@ -318,14 +323,7 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: "name": kea_statistic_name, }, ) - - responses = send_query(query, address, port, https, session=s) - if len(responses) != 1: - raise Exception(f"Received invalid amount of responses from '{address}'") # TODO: Change Exception - - response = responses[0] - if not response.success: - raise Exception("Did not receive statistics from DHCP server") + response = unwrap(send_query(query, address, port, https, session=s)) datapoints = response["arguments"].get(kea_statistic_name, []) for value, timestamp in datapoints: From 542038e4029d38c048a5b8500c95d6097451793f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 28 Jun 2024 17:54:03 +0200 Subject: [PATCH 18/77] Update comments Also fixes a typo on line 342 where an extra comma had sneaked itself into the code --- python/nav/dhcp/kea_dhcp_data.py | 61 ++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 6529660445..57e5ad4284 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -31,10 +31,11 @@ logger = logging.getLogger(__name__) class KeaError(GeneralException): + """Error related to interaction with a Kea Control Agent""" pass class KeaStatus(IntEnum): - """Status of a REST response.""" + """Status of a response sent from a Kea Control Agent.""" # Successful operation. SUCCESS = 0 # General failure. @@ -50,8 +51,8 @@ class KeaStatus(IntEnum): @dataclass class KeaResponse: """ - Class representing a REST response on a REST query sent to a Kea Control - Agent. + Class representing the response to a REST query sent to a Kea + Control Agent. """ result: int text: str @@ -72,13 +73,15 @@ class KeaQuery: def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: """ - Internal function. - Send `query` to a Kea Control Agent listening to `port` - on IP address `address`, using either http or https + Send `query` to a Kea Control Agent listening to `port` on IP + address `address`, using either http or https - :param session: optional session to be used when sending the query. Assumed - to not be closed. Session is not closed after the query, so that the session - can be used for persistent connections among differend send_query calls. + :param https: If True, use https. Otherwise, use http. + + :param session: Optional requests.Session to be used when sending + the query. Assumed to not be closed. session is not closed after + the end of this call, so that session can be used for persistent + http connections among different send_query calls. """ scheme = "https" if https else "http" location = f"{scheme}://{address}:{port}/" @@ -114,6 +117,10 @@ def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = Tru return responses def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: + """ + Helper function implementing the sequence of operations often done + on the list of responses returned by `send_query()` + """ if len(responses) != 1: raise KeaError(f"Received invalid amount of responses") @@ -243,12 +250,23 @@ def from_json( class KeaDhcpMetricSource(DhcpMetricSource): + """ + Using `send_query()`, this class: + * Maintains an up-to-date `KeaDhcpConfig` representation of the + configuration of the Kea DHCP server with ip version + `self.ip_version` reachable via the Kea Control Agent listening + to port `self.rest_port` on IP addresses `self.rest_address` + * Queries the Kea Control Agent for statistics about each subnet + found in the `KeaDhcpConfig` representation and creates an + iterable of `DhcpMetric` that its superclass uses to fill a + graphite server with metrics. + """ rest_address: str # IP address of the Kea Control Agent server rest_port: int # Port of the Kea Control Agent server rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. ip_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server - kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server that is used as a data/metric source + kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. def __init__(self, address: str, port: int, https: bool = True, ip_version: int = 4, *args, **kwargs): super(*args, **kwargs) @@ -260,9 +278,10 @@ def __init__(self, address: str, port: int, https: bool = True, ip_version: int def fetch_and_set_dhcp_config(self, session=None): """ - Fetch the config of the Kea DHCP server that manages addresses of IP - version `self.ip_version` from the Kea Control Agent listening to - `self.rest_port` on `self.rest_address`. + Fetch the current config used by the Kea DHCP server that + manages addresses of IP version `self.ip_version` from the Kea + Control Agent listening to `self.rest_port` on + `self.rest_address`. """ # Check if self.kea_dhcp_config is up to date if not ( @@ -284,6 +303,11 @@ def fetch_and_set_dhcp_config(self, session=None): return self.kea_dhcp_config def fetch_dhcp_config_hash(self, session=None): + """ + For Kea versions >= 2.4.0, fetch and return a hash of the + current configuration used by the Kea DHCP server. For Kea + versions < 2.4.0, return None. + """ query = KeaQuery( command="config-hash-get", service=[f"dhcp{self.ip_version}"], @@ -305,17 +329,17 @@ def fetch_dhcp_config_hash(self, session=None): def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: int = 4) -> list[DhcpMetric]: """ Implementation of the superclass method for fetching - standardised dhcp metrics; this method is what nav uses to - feed data into the graphite server. + standardised dhcp metrics. This method is used by the + superclass to feed data into a graphite server. """ metrics = [] - with requests.Session() as s: + with requests.Session() as s: # All possible exceptions (HTTPError, KeyError, JSONDecodeError, KeaError) will cause a breakout and then be ignored. Is this okay? self.fetch_and_set_dhcp_config(s) for subnet in self.kea_dhcp_config.subnets: for statistic_key, metric_key in (("total-addresses", DhcpMetricKey.MAX), ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH)): # dhcmetric_key is the same as the graphite metric names used in nav/contrib/scripts/isc_dhpcd_graphite/isc_dhpcd_graphite.py - kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}", + ("declined-addresses", DhcpMetricKey.TOUCH)): # `statistic_key` is the name of the statistic used by Kea. `metric_key` is the name of the statistic used by NAV. + kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}" query = KeaQuery( command="statistic-get", service=[f"dhcp{self.ip_version}"], @@ -330,7 +354,6 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! metrics.append(DhcpMetric(epochseconds, subnet.prefix, metric_key, value)) - used_config = self.kea_dhcp_config self.fetch_and_set_dhcp_config(s) if sorted(used_config.subnets) != sorted(self.kea_dhcp_config.subnets): From e6fbcf43008b29cd74b9c77f7a950bed9bc37fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 09:50:52 +0200 Subject: [PATCH 19/77] Find subnets defined in shared-networks section of Kea DHCP config What: Before, we only included subnets defined in the subnet[4,6] section of the Kea DHCP config obtained through the "config-get" query in the KeaDhcpConfig.subnets list. Now we also include the subnets defined in the shared-networks section of the Kea DHCP config, and thus we include all subnets than could possibly be configured for a Kea DHCP server, which means that we can now fetch metrics from all defined subnets. --- python/nav/dhcp/kea_dhcp_data.py | 21 ++++- tests/unittests/dhcp/kea_dhcp_data_test.py | 99 +++++++++++++++++++++- 2 files changed, 115 insertions(+), 5 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 57e5ad4284..ac251443f4 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -211,6 +211,21 @@ def from_json( Example: { "Dhcp4": { + "shared-networks": [ + { + "name": "test-network", + "subnet4": [ + { + "subnet": "10.0.0.0/8", + "pools": [ { "pool": "10.0.0.1 - 10.0.0.99" } ], + }, + { + "subnet": "192.0.3.0/24", + "pools": [ { "pool": "192.0.3.100 - 192.0.3.199" } ] + } + ], + } + ], # end of shared-networks "subnet4": [{ "id": 1, "subnet": "192.0.2.0/24", @@ -219,7 +234,7 @@ def from_json( "pool": "192.0.2.1 - 192.0.2.200", }, ], - }] + }] # end of subnet4 } } @@ -241,6 +256,10 @@ def from_json( for obj in json.get(f"subnet{ip_version}", []): subnet = KeaDhcpSubnet.from_json(obj) subnets.append(subnet) + for obj in json.get("shared-networks", []): + for subobj in obj.get(f"subnet{ip_version}", []): + subnet = KeaDhcpSubnet.from_json(subobj) + subnets.append(subnet) return cls( config_hash=config_hash, diff --git a/tests/unittests/dhcp/kea_dhcp_data_test.py b/tests/unittests/dhcp/kea_dhcp_data_test.py index d7e3743c06..4ac07207db 100644 --- a/tests/unittests/dhcp/kea_dhcp_data_test.py +++ b/tests/unittests/dhcp/kea_dhcp_data_test.py @@ -98,6 +98,49 @@ def dhcp4_config(): } ''' +@pytest.fixture +def dhcp4_config_w_shared_networks(): + return '''{ + "Dhcp4": { + "shared-networks": [ + { + "name": "shared-network-1", + "subnet4": [ + { + "id": 4, + "subnet": "10.0.0.0/8", + "pools": [ { "pool": "10.0.0.1 - 10.0.0.99" } ] + }, + { + "id": 3, + "subnet": "192.0.3.0/24", + "pools": [ { "pool": "192.0.3.100 - 192.0.3.199" } ] + } + ] + }, + { + "name": "shared-network-2", + "subnet4": [ + { + "id": 2, + "subnet": "192.0.2.0/24", + "pools": [ { "pool": "192.0.2.100 - 192.0.2.199" } ] + } + ] + } + ], + "subnet4": [{ + "id": 1, + "subnet": "192.0.1.0/24", + "pools": [ + { + "pool": "192.0.1.1 - 192.0.1.200" + } + ] + }] + } + }''' + @pytest.fixture(autouse=True) def enqueue_post_response(monkeypatch): """ @@ -267,7 +310,45 @@ def test_correct_config_from_dhcp4_config_json(dhcp4_config): assert config.ip_version == 4 assert config.config_hash is None -@pytest.fixture +def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_shared_networks): + j = json.loads(dhcp4_config_w_shared_networks) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 4 + subnets = {subnet.id: subnet for subnet in config.subnets} + + subnet1 = subnets[1] + assert subnet1.id == 1 + assert subnet1.prefix == IP("192.0.1.0/24") + assert len(subnet1.pools) == 1 + assert subnet1.pools[0] == (IP("192.0.1.1"), IP("192.0.1.200")) + assert config.ip_version == 4 + assert config.config_hash is None + + subnet2 = subnets[2] + assert subnet2.id == 2 + assert subnet2.prefix == IP("192.0.2.0/24") + assert len(subnet2.pools) == 1 + assert subnet2.pools[0] == (IP("192.0.2.100"), IP("192.0.2.199")) + assert config.ip_version == 4 + assert config.config_hash is None + + subnet3 = subnets[3] + assert subnet3.id == 3 + assert subnet3.prefix == IP("192.0.3.0/24") + assert len(subnet3.pools) == 1 + assert subnet3.pools[0] == (IP("192.0.3.100"), IP("192.0.3.199")) + assert config.ip_version == 4 + assert config.config_hash is None + + subnet4 = subnets[4] + assert subnet4.id == 4 + assert subnet4.prefix == IP("10.0.0.0/8") + assert len(subnet4.pools) == 1 + assert subnet4.pools[0] == (IP("10.0.0.1"), IP("10.0.0.99")) + assert config.ip_version == 4 + assert config.config_hash is None + + def dhcp4_config_response(dhcp4_config): return f''' [ @@ -284,15 +365,25 @@ def dhcp4_config_response(dhcp4_config): # Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from server responses # ################################################################################ -def test_fetch_and_set_dhcp_config(dhcp4_config_response, enqueue_post_response): - enqueue_post_response("config-get", dhcp4_config_response) +def test_fetch_and_set_dhcp_config(dhcp4_config, enqueue_post_response): + enqueue_post_response("config-get", dhcp4_config_response(dhcp4_config)) source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) assert source.kea_dhcp_config is None config = source.fetch_and_set_dhcp_config() - actual_config = KeaDhcpConfig.from_json(json.loads(dhcp4_config_response)[0]["arguments"]) + actual_config = KeaDhcpConfig.from_json(json.loads(dhcp4_config_response(dhcp4_config))[0]["arguments"]) assert config == actual_config assert source.kea_dhcp_config == actual_config +def test_fetch_and_set_dhcp_config_w_shared_networks(dhcp4_config_w_shared_networks, enqueue_post_response): + enqueue_post_response("config-get", dhcp4_config_response(dhcp4_config_w_shared_networks)) + source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) + assert source.kea_dhcp_config is None + config = source.fetch_and_set_dhcp_config() + actual_config = KeaDhcpConfig.from_json(json.loads(dhcp4_config_response(dhcp4_config_w_shared_networks))[0]["arguments"]) + assert config == actual_config + assert source.kea_dhcp_config == actual_config + + # @pytest.fixture # @enqueue_post_response # def dhcp4_config_response_result_is_1(): From 3a462ace0af4469916ac1e10e13a40ceff41c40b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 09:55:15 +0200 Subject: [PATCH 20/77] Add exception handling in the fetch_metrics method of KeaDhcpMetricSource --- python/nav/dhcp/kea_dhcp_data.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index ac251443f4..83cc73291a 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -352,7 +352,8 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: superclass to feed data into a graphite server. """ metrics = [] - with requests.Session() as s: # All possible exceptions (HTTPError, KeyError, JSONDecodeError, KeaError) will cause a breakout and then be ignored. Is this okay? + try: + s = requests.Session() self.fetch_and_set_dhcp_config(s) for subnet in self.kea_dhcp_config.subnets: for statistic_key, metric_key in (("total-addresses", DhcpMetricKey.MAX), @@ -381,5 +382,29 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: "this may cause metric data being associated with wrong " "subnet." ) + except HTTPError as err: + logger.warning( + "HTTPError while fetching metrics from Kea Control Agent with url %s", + "(unknown)" if err.response is None else err.response.url, + exc_info=err, + ) + pass + except KeaError as err: + logger.warning( + "KeaError while fetching metrics from Kea Control Agent", + exc_info=err, + ) + except KeyError as err: + logging.warning( + "KeyError while fetching metrics from Kea Control Agent", + exc_info=err, + ) + except JSONDecodeError as err: + logger.warning( + "JSONDecodeError while fetching metrics from Kea Control Agent", + exc_info=err, + ) + finally: + s.close() return metrics From ae12df19a412c6b7c5ef61df19cb539507289a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 10:27:19 +0200 Subject: [PATCH 21/77] Use instance variables instead of function parameters in fetch_metrics Why: This is necessary to match the fetch_metrics definition of the superclass --- python/nav/dhcp/kea_dhcp_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 83cc73291a..13a06b726f 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -345,7 +345,7 @@ def fetch_dhcp_config_hash(self, session=None): else: raise KeaError("Unexpected error when querying the hash of config file from DHCP server") - def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: int = 4) -> list[DhcpMetric]: + def fetch_metrics(self) -> list[DhcpMetric]: """ Implementation of the superclass method for fetching standardised dhcp metrics. This method is used by the @@ -367,7 +367,7 @@ def fetch_metrics(self, address: str, port: int, https: bool = True, ip_version: "name": kea_statistic_name, }, ) - response = unwrap(send_query(query, address, port, https, session=s)) + response = unwrap(send_query(query, self.rest_address, self.rest_port, self.rest_https, session=s)) datapoints = response["arguments"].get(kea_statistic_name, []) for value, timestamp in datapoints: From 6775dd9338fff6d77bc38c3a3e74a9e833e982f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 10:32:22 +0200 Subject: [PATCH 22/77] Change ValueErrors to KeaErrors Why: For some of the errors, it makes more sense for them to remain as ValueErrors because they are caused by an error during parameter checks, but I've switched them to KeaErrors because they are still also viable as being KeaErrors and this change simplifies exception handling. --- python/nav/dhcp/kea_dhcp_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 13a06b726f..d4b2c7c33d 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -158,11 +158,11 @@ def from_json(cls, subnet_json: dict): } """ if "id" not in subnet_json: - raise ValueError("Expected subnetjson['id'] to exist") + raise KeaError("Expected subnetjson['id'] to exist") id = subnet_json["id"] if "subnet" not in subnet_json: - raise ValueError("Expected subnetjson['subnet'] to exist") + raise KeaError("Expected subnetjson['subnet'] to exist") prefix = IP(subnet_json["subnet"]) pools = [] @@ -242,7 +242,7 @@ def from_json( `config-hash-get` query on the kea-ctrl-agent REST server. """ if len(config_json) > 1: - raise ValueError("Did not expect len(configjson) > 1") + raise KeaError("Did not expect len(configjson) > 1") ip_version, json = config_json.popitem() if ip_version == "Dhcp4": @@ -250,7 +250,7 @@ def from_json( elif ip_version == "Dhcp6": ip_version == 6 else: - raise ValueError(f"Unsupported DHCP IP version '{ip_version}'") + raise KeaError(f"Unsupported DHCP IP version '{ip_version}'") subnets = [] for obj in json.get(f"subnet{ip_version}", []): From 71fdf4108177f77b729425295c092e56cdb290af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 10:40:26 +0200 Subject: [PATCH 23/77] Be less strict when typing JSON dicts What: Json dicts can be nested, and we need to make sure any typing for json dicts accounts for that. The previous typing efforts did not, and for now, instead of defining a perfect json type, we'll just be more lenient with the json dict type, allowing any valid python dicts. --- python/nav/dhcp/kea_dhcp_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index d4b2c7c33d..3bb97d75e5 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -56,7 +56,7 @@ class KeaResponse: """ result: int text: str - arguments: dict[str: Union[str, int]] + arguments: dict service: str @property @@ -68,7 +68,7 @@ def success(self): class KeaQuery: """Class representing a REST query to be sent to a Kea Control Agent.""" command: str - arguments: dict[str: Union[str, int]] + arguments: dict service: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: From 302553e9b7912c12c823e24ad69762b1d92651e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 10:57:55 +0200 Subject: [PATCH 24/77] Exception handlers use `logger.debug` if they reraise exception What: To hinder occurrances of duplicate logging of a raised exception, the convention where only the last handler of an exception (the handler that doesn't reraise the exception) logs with a logging level higher than DEBUG is used. This makes it so that deeper exception handlers that are able to give more details about the exception can still use logging, but they only log at the DEBUG level. This pairs well with the idea that the lower the logging level, the more details one will get. --- python/nav/dhcp/kea_dhcp_data.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 3bb97d75e5..dfdd759d73 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -60,7 +60,7 @@ class KeaResponse: service: str @property - def success(self): + def success(self) -> bool: return self.result == KeaStatus.SUCCESS @@ -92,17 +92,17 @@ def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = Tru else: r = session.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) except HTTPError as err: - logger.error("send_query: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) + logger.debug("send_query: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) # non-debug logging done by exception handler raise err try: response_json = r.json() except JSONDecodeError as err: - logger.error("send_query: expected json from %s, got %s", address, r.text) + logger.debug("send_query: expected json from %s, got %s", address, r.text) # non-debug logging done by exception handler raise err if isinstance(response_json, dict): - logger.error("send_query: expected a json list of objects from %s, got %r", address, response_json) + logger.debug("send_query: expected a json list of objects from %s, got %r", address, response_json) # non-debug logging done by exception handler raise KeaError(f"bad response from {address}: {response_json!r}") responses = [] @@ -188,9 +188,7 @@ class KeaDhcpConfig: """ Class representing information found in the configuration of a Kea DHCP server. Most importantly, this class contains: - * A list of the shared networks managed by the DHCP server - * A shared network furthermore contain a list of subnets - assigned to that network + * A list of the subnets managed by the DHCP server * The IP version of the DCHP server """ config_hash: Optional[str] # Used to check if there's a new config on the Kea DHCP server @@ -204,7 +202,7 @@ def from_json( config_hash: Optional[str] = None, ): """ - Initialize and return a KeaDhcpData instance based on json + Initialize and return a KeaDhcpConfig instance based on json :param json: a dictionary that is structured the same way as a Kea DHCP configuration. @@ -343,7 +341,7 @@ def fetch_dhcp_config_hash(self, session=None): elif response.success: return response.arguments.get("hash", None) else: - raise KeaError("Unexpected error when querying the hash of config file from DHCP server") + raise KeaError("Unexpected error while querying the hash of config file from DHCP server") def fetch_metrics(self) -> list[DhcpMetric]: """ @@ -388,14 +386,13 @@ def fetch_metrics(self) -> list[DhcpMetric]: "(unknown)" if err.response is None else err.response.url, exc_info=err, ) - pass except KeaError as err: logger.warning( "KeaError while fetching metrics from Kea Control Agent", exc_info=err, ) except KeyError as err: - logging.warning( + logger.warning( "KeyError while fetching metrics from Kea Control Agent", exc_info=err, ) From 0d69b065f1dba34434d3367bffbaddcd68a1e89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 11:59:16 +0200 Subject: [PATCH 25/77] Add timeouts to http requests --- python/nav/dhcp/kea_dhcp_data.py | 41 ++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index dfdd759d73..e9cee4487b 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -88,11 +88,24 @@ def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = Tru logger.debug("send_query: sending request to %s with query %r", location, query) try: if session is None: - r = requests.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) + r = requests.post( + location, + data=json.dumps(asdict(query)), + headers={"Content-Type": "application/json"}, + timeout=timeout + ) else: - r = session.post(location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}) + r = session.post( + location, + data=json.dumps(asdict(query)), + headers={"Content-Type": "application/json"}, + timeout=timeout + ) except HTTPError as err: - logger.debug("send_query: request to %s yielded an error: %d %s", err.url, err.status_code, err.reason) # non-debug logging done by exception handler + logger.debug("send_query: request to %s yielded an error: %d %s", err.request.url, err.response.status_code, err.response.reason) # non-debug logging done by exception handler + raise err + except Timeout as err: + logger.debug("send_query: request to %s timed out", err.request.url) # non-debug logging done by exception handler raise err try: @@ -380,25 +393,17 @@ def fetch_metrics(self) -> list[DhcpMetric]: "this may cause metric data being associated with wrong " "subnet." ) - except HTTPError as err: - logger.warning( - "HTTPError while fetching metrics from Kea Control Agent with url %s", - "(unknown)" if err.response is None else err.response.url, - exc_info=err, - ) - except KeaError as err: - logger.warning( - "KeaError while fetching metrics from Kea Control Agent", - exc_info=err, - ) - except KeyError as err: + except Timeout as err: logger.warning( - "KeyError while fetching metrics from Kea Control Agent", + "Connection to Kea Control Agent timed before or during metric " + "fetching. Some metrics may be missing.", exc_info=err, ) - except JSONDecodeError as err: + except Exception as err: + # More detailed information is logged by deeper exception handlers at the logging.DEBUG level. logger.warning( - "JSONDecodeError while fetching metrics from Kea Control Agent", + "Exception while fetching metrics from Kea Control Agent. Some " + "metrics may be missing.", exc_info=err, ) finally: From d4b14c2b21b47c130a6ab9e4aa8e41df83c4983c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 11:59:27 +0200 Subject: [PATCH 26/77] Fix (most) pylint errors --- python/nav/dhcp/kea_dhcp_data.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index e9cee4487b..93467328de 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -21,18 +21,17 @@ import requests import time from dataclasses import dataclass, asdict -from .dhcp_data import DhcpMetricSource, DhcpMetric, DhcpMetricKey from enum import IntEnum from IPy import IP +from nav.dhcp.dhcp_data import DhcpMetricSource, DhcpMetric, DhcpMetricKey from nav.errors import GeneralException -from requests.exceptions import JSONDecodeError, HTTPError -from typing import Union, Optional +from requests.exceptions import JSONDecodeError, HTTPError, Timeout +from typing import Optional logger = logging.getLogger(__name__) class KeaError(GeneralException): """Error related to interaction with a Kea Control Agent""" - pass class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" @@ -71,7 +70,14 @@ class KeaQuery: arguments: dict service: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. -def send_query(query: KeaQuery, address: str, port: int = 443, https: bool = True, session: requests.Session = None) -> list[KeaResponse]: +def send_query( + query: KeaQuery, + address: str, + port: int = 443, + https: bool = True, + session: requests.Session = None, + timeout: int = 10, +) -> list[KeaResponse]: """ Send `query` to a Kea Control Agent listening to `port` on IP address `address`, using either http or https @@ -135,7 +141,7 @@ def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: on the list of responses returned by `send_query()` """ if len(responses) != 1: - raise KeaError(f"Received invalid amount of responses") + raise KeaError("Received invalid amount of responses") response = responses[0] if require_success and not response.success: @@ -255,19 +261,19 @@ def from_json( if len(config_json) > 1: raise KeaError("Did not expect len(configjson) > 1") - ip_version, json = config_json.popitem() + ip_version, config_json = config_json.popitem() if ip_version == "Dhcp4": ip_version = 4 elif ip_version == "Dhcp6": - ip_version == 6 + ip_version = 6 else: raise KeaError(f"Unsupported DHCP IP version '{ip_version}'") subnets = [] - for obj in json.get(f"subnet{ip_version}", []): + for obj in config_json.get(f"subnet{ip_version}", []): subnet = KeaDhcpSubnet.from_json(obj) subnets.append(subnet) - for obj in json.get("shared-networks", []): + for obj in config_json.get("shared-networks", []): for subobj in obj.get(f"subnet{ip_version}", []): subnet = KeaDhcpSubnet.from_json(subobj) subnets.append(subnet) @@ -298,7 +304,7 @@ class KeaDhcpMetricSource(DhcpMetricSource): ip_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. - def __init__(self, address: str, port: int, https: bool = True, ip_version: int = 4, *args, **kwargs): + def __init__(self, address: str, port: int, *args, https: bool = True, ip_version: int = 4, **kwargs): super(*args, **kwargs) self.rest_address = address self.rest_port = port @@ -317,7 +323,7 @@ def fetch_and_set_dhcp_config(self, session=None): if not ( self.kea_dhcp_config is None or self.kea_dhcp_config.config_hash is None - or self.fetch_dhcp_config_hash(session=s) != self.kea_dhcp_config.config_hash + or self.fetch_dhcp_config_hash(session=session) != self.kea_dhcp_config.config_hash ): return self.kea_dhcp_config @@ -349,7 +355,7 @@ def fetch_dhcp_config_hash(self, session=None): ) if response.result == KeaStatus.UNSUPPORTED: - logger.info("Kea DHCP%d server does not support quering for the hash of its config", ip_version) + logger.info("Kea DHCP%d server does not support quering for the hash of its config", self.ip_version) return None elif response.success: return response.arguments.get("hash", None) From 87d90c5da85faf3c149cfca4198fb4b9b5a82582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 12:01:41 +0200 Subject: [PATCH 27/77] Remove unnessecary comments --- python/nav/dhcp/kea_dhcp_data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_dhcp_data.py index 93467328de..f0baec2b92 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_dhcp_data.py @@ -108,20 +108,20 @@ def send_query( timeout=timeout ) except HTTPError as err: - logger.debug("send_query: request to %s yielded an error: %d %s", err.request.url, err.response.status_code, err.response.reason) # non-debug logging done by exception handler + logger.debug("send_query: request to %s yielded an error: %d %s", err.request.url, err.response.status_code, err.response.reason) raise err except Timeout as err: - logger.debug("send_query: request to %s timed out", err.request.url) # non-debug logging done by exception handler + logger.debug("send_query: request to %s timed out", err.request.url) raise err try: response_json = r.json() except JSONDecodeError as err: - logger.debug("send_query: expected json from %s, got %s", address, r.text) # non-debug logging done by exception handler + logger.debug("send_query: expected json from %s, got %s", address, r.text) raise err if isinstance(response_json, dict): - logger.debug("send_query: expected a json list of objects from %s, got %r", address, response_json) # non-debug logging done by exception handler + logger.debug("send_query: expected a json list of objects from %s, got %r", address, response_json) raise KeaError(f"bad response from {address}: {response_json!r}") responses = [] @@ -406,7 +406,7 @@ def fetch_metrics(self) -> list[DhcpMetric]: exc_info=err, ) except Exception as err: - # More detailed information is logged by deeper exception handlers at the logging.DEBUG level. + # More detailed information should be logged by deeper exception handlers at the logging.DEBUG level. logger.warning( "Exception while fetching metrics from Kea Control Agent. Some " "metrics may be missing.", From 0f81e5e6a6b657514e0ef9f95927a72cb25727a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 13:38:45 +0200 Subject: [PATCH 28/77] Rename kea_dchp_data.py to kea_metrics.py --- python/nav/dhcp/{dhcp_data.py => generic_metrics.py} | 0 python/nav/dhcp/{kea_dhcp_data.py => kea_metrics.py} | 2 +- tests/unittests/dhcp/kea_dhcp_data_test.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename python/nav/dhcp/{dhcp_data.py => generic_metrics.py} (100%) rename python/nav/dhcp/{kea_dhcp_data.py => kea_metrics.py} (99%) diff --git a/python/nav/dhcp/dhcp_data.py b/python/nav/dhcp/generic_metrics.py similarity index 100% rename from python/nav/dhcp/dhcp_data.py rename to python/nav/dhcp/generic_metrics.py diff --git a/python/nav/dhcp/kea_dhcp_data.py b/python/nav/dhcp/kea_metrics.py similarity index 99% rename from python/nav/dhcp/kea_dhcp_data.py rename to python/nav/dhcp/kea_metrics.py index f0baec2b92..faf61cd4a1 100644 --- a/python/nav/dhcp/kea_dhcp_data.py +++ b/python/nav/dhcp/kea_metrics.py @@ -23,7 +23,7 @@ from dataclasses import dataclass, asdict from enum import IntEnum from IPy import IP -from nav.dhcp.dhcp_data import DhcpMetricSource, DhcpMetric, DhcpMetricKey +from nav.dhcp.generic_metrics import DhcpMetricSource, DhcpMetric, DhcpMetricKey from nav.errors import GeneralException from requests.exceptions import JSONDecodeError, HTTPError, Timeout from typing import Optional diff --git a/tests/unittests/dhcp/kea_dhcp_data_test.py b/tests/unittests/dhcp/kea_dhcp_data_test.py index 4ac07207db..50cb60fa5a 100644 --- a/tests/unittests/dhcp/kea_dhcp_data_test.py +++ b/tests/unittests/dhcp/kea_dhcp_data_test.py @@ -1,5 +1,5 @@ from collections import deque -from nav.dhcp.kea_dhcp_data import * +from nav.dhcp.kea_metrics import * import pytest import requests from IPy import IP From 6d5242a096d6b241f4f07787658d1cfcfc1ddd70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 14:15:02 +0200 Subject: [PATCH 29/77] Use dhcp_version instead of ip_version to avoid ambiguities What: Previously, classes and functions used a parameter called `ip_version` for specifying e.g. which of the two possible dhcp-clients we want to fetch metrics from in KeaDhcpMetricSource. To make intentions of the parameter (that is, to specify if we want to query the Kea DHCP4 server or the Kea DHCP6 server) more clear, the `ip_version` parameter names are changed to `dhcp_version`. Otherwise, by looking at the function signature, one might think that the intention of `ip_version` is to specify what ip version the parameter `address` uses. --- python/nav/dhcp/kea_metrics.py | 38 +++++++++---------- ..._dhcp_data_test.py => kea_metrics_test.py} | 12 +++--- 2 files changed, 25 insertions(+), 25 deletions(-) rename tests/unittests/dhcp/{kea_dhcp_data_test.py => kea_metrics_test.py} (98%) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index faf61cd4a1..a6758a1dfa 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -211,7 +211,7 @@ class KeaDhcpConfig: * The IP version of the DCHP server """ config_hash: Optional[str] # Used to check if there's a new config on the Kea DHCP server - ip_version: int + dhcp_version: int subnets: list[KeaDhcpSubnet] @classmethod @@ -261,26 +261,26 @@ def from_json( if len(config_json) > 1: raise KeaError("Did not expect len(configjson) > 1") - ip_version, config_json = config_json.popitem() - if ip_version == "Dhcp4": - ip_version = 4 - elif ip_version == "Dhcp6": - ip_version = 6 + dhcp_version, config_json = config_json.popitem() + if dhcp_version == "Dhcp4": + dhcp_version = 4 + elif dhcp_version == "Dhcp6": + dhcp_version = 6 else: - raise KeaError(f"Unsupported DHCP IP version '{ip_version}'") + raise KeaError(f"Unsupported DHCP IP version '{dhcp_version}'") subnets = [] - for obj in config_json.get(f"subnet{ip_version}", []): + for obj in config_json.get(f"subnet{dhcp_version}", []): subnet = KeaDhcpSubnet.from_json(obj) subnets.append(subnet) for obj in config_json.get("shared-networks", []): - for subobj in obj.get(f"subnet{ip_version}", []): + for subobj in obj.get(f"subnet{dhcp_version}", []): subnet = KeaDhcpSubnet.from_json(subobj) subnets.append(subnet) return cls( config_hash=config_hash, - ip_version=ip_version, + dhcp_version=dhcp_version, subnets=subnets, ) @@ -290,7 +290,7 @@ class KeaDhcpMetricSource(DhcpMetricSource): Using `send_query()`, this class: * Maintains an up-to-date `KeaDhcpConfig` representation of the configuration of the Kea DHCP server with ip version - `self.ip_version` reachable via the Kea Control Agent listening + `self.dhcp_version` reachable via the Kea Control Agent listening to port `self.rest_port` on IP addresses `self.rest_address` * Queries the Kea Control Agent for statistics about each subnet found in the `KeaDhcpConfig` representation and creates an @@ -301,21 +301,21 @@ class KeaDhcpMetricSource(DhcpMetricSource): rest_port: int # Port of the Kea Control Agent server rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. - ip_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server + dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. - def __init__(self, address: str, port: int, *args, https: bool = True, ip_version: int = 4, **kwargs): + def __init__(self, address: str, port: int, *args, https: bool = True, dhcp_version: int = 4, **kwargs): super(*args, **kwargs) self.rest_address = address self.rest_port = port self.rest_https = https - self.ip_version = ip_version + self.dchp_version = dhcp_version self.kea_dhcp_config = None def fetch_and_set_dhcp_config(self, session=None): """ Fetch the current config used by the Kea DHCP server that - manages addresses of IP version `self.ip_version` from the Kea + manages addresses of IP version `self.dhcp_version` from the Kea Control Agent listening to `self.rest_port` on `self.rest_address`. """ @@ -330,7 +330,7 @@ def fetch_and_set_dhcp_config(self, session=None): # self.kea_dhcp_config is not up to date, fetch new query = KeaQuery( command="config-get", - service=[f"dhcp{self.ip_version}"], + service=[f"dhcp{self.dchp_version}"], arguments={}, ) response = unwrap(send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session)) @@ -346,7 +346,7 @@ def fetch_dhcp_config_hash(self, session=None): """ query = KeaQuery( command="config-hash-get", - service=[f"dhcp{self.ip_version}"], + service=[f"dhcp{self.dchp_version}"], arguments={}, ) response = unwrap( @@ -355,7 +355,7 @@ def fetch_dhcp_config_hash(self, session=None): ) if response.result == KeaStatus.UNSUPPORTED: - logger.info("Kea DHCP%d server does not support quering for the hash of its config", self.ip_version) + logger.info("Kea DHCP%d server does not support quering for the hash of its config", self.dchp_version) return None elif response.success: return response.arguments.get("hash", None) @@ -379,7 +379,7 @@ def fetch_metrics(self) -> list[DhcpMetric]: kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}" query = KeaQuery( command="statistic-get", - service=[f"dhcp{self.ip_version}"], + service=[f"dhcp{self.dchp_version}"], arguments={ "name": kea_statistic_name, }, diff --git a/tests/unittests/dhcp/kea_dhcp_data_test.py b/tests/unittests/dhcp/kea_metrics_test.py similarity index 98% rename from tests/unittests/dhcp/kea_dhcp_data_test.py rename to tests/unittests/dhcp/kea_metrics_test.py index 50cb60fa5a..39eb498303 100644 --- a/tests/unittests/dhcp/kea_dhcp_data_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -307,7 +307,7 @@ def test_correct_config_from_dhcp4_config_json(dhcp4_config): assert len(subnet.pools) == 2 assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) - assert config.ip_version == 4 + assert config.dhcp_version == 4 assert config.config_hash is None def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_shared_networks): @@ -321,7 +321,7 @@ def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_ assert subnet1.prefix == IP("192.0.1.0/24") assert len(subnet1.pools) == 1 assert subnet1.pools[0] == (IP("192.0.1.1"), IP("192.0.1.200")) - assert config.ip_version == 4 + assert config.dhcp_version == 4 assert config.config_hash is None subnet2 = subnets[2] @@ -329,7 +329,7 @@ def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_ assert subnet2.prefix == IP("192.0.2.0/24") assert len(subnet2.pools) == 1 assert subnet2.pools[0] == (IP("192.0.2.100"), IP("192.0.2.199")) - assert config.ip_version == 4 + assert config.dhcp_version == 4 assert config.config_hash is None subnet3 = subnets[3] @@ -337,7 +337,7 @@ def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_ assert subnet3.prefix == IP("192.0.3.0/24") assert len(subnet3.pools) == 1 assert subnet3.pools[0] == (IP("192.0.3.100"), IP("192.0.3.199")) - assert config.ip_version == 4 + assert config.dhcp_version == 4 assert config.config_hash is None subnet4 = subnets[4] @@ -345,7 +345,7 @@ def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_ assert subnet4.prefix == IP("10.0.0.0/8") assert len(subnet4.pools) == 1 assert subnet4.pools[0] == (IP("10.0.0.1"), IP("10.0.0.99")) - assert config.ip_version == 4 + assert config.dhcp_version == 4 assert config.config_hash is None @@ -398,4 +398,4 @@ def test_fetch_and_set_dhcp_config_w_shared_networks(dhcp4_config_w_shared_netwo # def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): # with pytest.raises(Exception): # TODO: Change -# get_dhcp_server("example-org", ip_version=4) +# get_dhcp_server("example-org", dhcp_version=4) From 363971cf8afca86e5fdc90d4e524afdb6d51e74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 1 Jul 2024 16:03:35 +0200 Subject: [PATCH 30/77] Fix linting errors --- python/nav/dhcp/generic_metrics.py | 15 ++- python/nav/dhcp/kea_metrics.py | 145 +++++++++++++++++------ tests/unittests/dhcp/kea_metrics_test.py | 52 ++++++-- 3 files changed, 158 insertions(+), 54 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 9471b53018..b427907dbc 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -4,13 +4,15 @@ from nav.metrics import carbon from typing import Iterator + class DhcpMetricKey(Enum): - MAX = "max" # total addresses - CUR = "cur" # assigned addresses - TOUCH = "touch" # touched addresses + MAX = "max" # total addresses + CUR = "cur" # assigned addresses + TOUCH = "touch" # touched addresses def __str__(self): - return self.name.lower() # For use in graphite path + return self.name.lower() # For use in graphite path + @dataclass class DhcpMetric: @@ -19,12 +21,14 @@ class DhcpMetric: key: DhcpMetricKey value: int + class DhcpMetricSource: """ Superclass for all classes that wish to collect metrics from a specific line of DHCP servers and import the metrics into NAV's graphite server. Subclasses need to implement `fetch_metrics`. """ + graphite_prefix: str def __init__(self, graphite_prefix="nav.dhcp"): @@ -36,8 +40,9 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: for each subnet of the DHCP server at current point of time. """ raise NotImplementedError + def fetch_metrics_to_graphite(self, host, port): - fmt = str.maketrans({".": "_", "/": "_"}) # 192.0.2.0/24 --> 192_0_0_0_24 + fmt = str.maketrans({".": "_", "/": "_"}) # 192.0.2.0/24 --> 192_0_0_0_24 graphite_metrics = [] for metric in self.fetch_metrics(): graphite_path = f"{self.graphite_prefix}.{str(metric.subnet_prefix).translate(fmt)}.{metric.key}" diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index a6758a1dfa..b049120153 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -15,6 +15,7 @@ * See also the Kea Control Agent documentation. This script assumes Kea versions >= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). """ + import calendar import json import logging @@ -30,11 +31,14 @@ logger = logging.getLogger(__name__) + class KeaError(GeneralException): """Error related to interaction with a Kea Control Agent""" + class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" + # Successful operation. SUCCESS = 0 # General failure. @@ -53,6 +57,7 @@ class KeaResponse: Class representing the response to a REST query sent to a Kea Control Agent. """ + result: int text: str arguments: dict @@ -66,17 +71,21 @@ def success(self) -> bool: @dataclass class KeaQuery: """Class representing a REST query to be sent to a Kea Control Agent.""" + command: str arguments: dict - service: list[str] # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. + + # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. + service: list[str] + def send_query( - query: KeaQuery, - address: str, - port: int = 443, - https: bool = True, - session: requests.Session = None, - timeout: int = 10, + query: KeaQuery, + address: str, + port: int = 443, + https: bool = True, + session: requests.Session = None, + timeout: int = 10, ) -> list[KeaResponse]: """ Send `query` to a Kea Control Agent listening to `port` on IP @@ -98,17 +107,22 @@ def send_query( location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}, - timeout=timeout + timeout=timeout, ) else: r = session.post( location, data=json.dumps(asdict(query)), headers={"Content-Type": "application/json"}, - timeout=timeout + timeout=timeout, ) except HTTPError as err: - logger.debug("send_query: request to %s yielded an error: %d %s", err.request.url, err.response.status_code, err.response.reason) + logger.debug( + "send_query: request to %s yielded an error: %d %s", + err.request.url, + err.response.status_code, + err.response.reason, + ) raise err except Timeout as err: logger.debug("send_query: request to %s timed out", err.request.url) @@ -121,7 +135,11 @@ def send_query( raise err if isinstance(response_json, dict): - logger.debug("send_query: expected a json list of objects from %s, got %r", address, response_json) + logger.debug( + "send_query: expected a json list of objects from %s, got %r", + address, + response_json, + ) raise KeaError(f"bad response from {address}: {response_json!r}") responses = [] @@ -135,6 +153,7 @@ def send_query( responses.append(response) return responses + def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: """ Helper function implementing the sequence of operations often done @@ -148,12 +167,14 @@ def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: raise KeaError("Did not receive a successful response") return response + @dataclass class KeaDhcpSubnet: """Class representing information about a subnet managed by a Kea DHCP server.""" - id: int # either specified in the server config or assigned automatically by the dhcp server - prefix: IP # e.g. 192.0.2.1/24 - pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] + + id: int # either specified in the server config or assigned automatically by the dhcp server + prefix: IP # e.g. 192.0.2.1/24 + pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] @classmethod def from_json(cls, subnet_json: dict): @@ -187,7 +208,7 @@ def from_json(cls, subnet_json: dict): pools = [] for obj in subnet_json.get("pools", []): pool = obj["pool"] - if "-" in pool: # TODO: Error checking? + if "-" in pool: # TODO: Error checking? # pool == "x.x.x.x - y.y.y.y" start, end = (IP(ip) for ip in pool.split("-")) else: @@ -202,6 +223,7 @@ def from_json(cls, subnet_json: dict): pools=pools, ) + @dataclass class KeaDhcpConfig: """ @@ -210,15 +232,17 @@ class KeaDhcpConfig: * A list of the subnets managed by the DHCP server * The IP version of the DCHP server """ - config_hash: Optional[str] # Used to check if there's a new config on the Kea DHCP server + + # Used to check if there's a new config on the Kea DHCP server + config_hash: Optional[str] dhcp_version: int subnets: list[KeaDhcpSubnet] @classmethod def from_json( - cls, - config_json: dict, - config_hash: Optional[str] = None, + cls, + config_json: dict, + config_hash: Optional[str] = None, ): """ Initialize and return a KeaDhcpConfig instance based on json @@ -297,14 +321,23 @@ class KeaDhcpMetricSource(DhcpMetricSource): iterable of `DhcpMetric` that its superclass uses to fill a graphite server with metrics. """ - rest_address: str # IP address of the Kea Control Agent server - rest_port: int # Port of the Kea Control Agent server - rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. - dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server - kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. + rest_address: str # IP address of the Kea Control Agent server + rest_port: int # Port of the Kea Control Agent server + rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. + + dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server + kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. - def __init__(self, address: str, port: int, *args, https: bool = True, dhcp_version: int = 4, **kwargs): + def __init__( + self, + address: str, + port: int, + *args, + https: bool = True, + dhcp_version: int = 4, + **kwargs, + ): super(*args, **kwargs) self.rest_address = address self.rest_port = port @@ -321,9 +354,10 @@ def fetch_and_set_dhcp_config(self, session=None): """ # Check if self.kea_dhcp_config is up to date if not ( - self.kea_dhcp_config is None - or self.kea_dhcp_config.config_hash is None - or self.fetch_dhcp_config_hash(session=session) != self.kea_dhcp_config.config_hash + self.kea_dhcp_config is None + or self.kea_dhcp_config.config_hash is None + or self.fetch_dhcp_config_hash(session=session) + != self.kea_dhcp_config.config_hash ): return self.kea_dhcp_config @@ -333,7 +367,15 @@ def fetch_and_set_dhcp_config(self, session=None): service=[f"dhcp{self.dchp_version}"], arguments={}, ) - response = unwrap(send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session)) + response = unwrap( + send_query( + query, + self.rest_address, + self.rest_port, + self.rest_https, + session=session, + ) + ) self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) return self.kea_dhcp_config @@ -350,17 +392,28 @@ def fetch_dhcp_config_hash(self, session=None): arguments={}, ) response = unwrap( - send_query(query, self.rest_address, self.rest_port, self.rest_https, session=session), - require_success=False + send_query( + query, + self.rest_address, + self.rest_port, + self.rest_https, + session=session, + ), + require_success=False, ) if response.result == KeaStatus.UNSUPPORTED: - logger.info("Kea DHCP%d server does not support quering for the hash of its config", self.dchp_version) + logger.info( + "Kea DHCP%d server does not support quering for the hash of its config", + self.dchp_version, + ) return None elif response.success: return response.arguments.get("hash", None) else: - raise KeaError("Unexpected error while querying the hash of config file from DHCP server") + raise KeaError( + "Unexpected error while querying the hash of config file from DHCP server" + ) def fetch_metrics(self) -> list[DhcpMetric]: """ @@ -373,9 +426,11 @@ def fetch_metrics(self) -> list[DhcpMetric]: s = requests.Session() self.fetch_and_set_dhcp_config(s) for subnet in self.kea_dhcp_config.subnets: - for statistic_key, metric_key in (("total-addresses", DhcpMetricKey.MAX), - ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH)): # `statistic_key` is the name of the statistic used by Kea. `metric_key` is the name of the statistic used by NAV. + for statistic_key, metric_key in ( + ("total-addresses", DhcpMetricKey.MAX), + ("assigned-addresses", DhcpMetricKey.CUR), + ("declined-addresses", DhcpMetricKey.TOUCH), + ): # `statistic_key` is the name of the statistic used by Kea. `metric_key` is the name of the statistic used by NAV. kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}" query = KeaQuery( command="statistic-get", @@ -384,12 +439,24 @@ def fetch_metrics(self) -> list[DhcpMetric]: "name": kea_statistic_name, }, ) - response = unwrap(send_query(query, self.rest_address, self.rest_port, self.rest_https, session=s)) + response = unwrap( + send_query( + query, + self.rest_address, + self.rest_port, + self.rest_https, + session=s, + ) + ) datapoints = response["arguments"].get(kea_statistic_name, []) for value, timestamp in datapoints: - epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! - metrics.append(DhcpMetric(epochseconds, subnet.prefix, metric_key, value)) + epochseconds = calendar.timegm( + time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + ) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! + metrics.append( + DhcpMetric(epochseconds, subnet.prefix, metric_key, value) + ) used_config = self.kea_dhcp_config self.fetch_and_set_dhcp_config(s) diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 39eb498303..6bfc291f0f 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -6,6 +6,7 @@ import json from requests.exceptions import JSONDecodeError + @pytest.fixture def dhcp4_config(): return ''' @@ -98,6 +99,7 @@ def dhcp4_config(): } ''' + @pytest.fixture def dhcp4_config_w_shared_networks(): return '''{ @@ -141,6 +143,7 @@ def dhcp4_config_w_shared_networks(): } }''' + @pytest.fixture(autouse=True) def enqueue_post_response(monkeypatch): """ @@ -152,7 +155,9 @@ def enqueue_post_response(monkeypatch): This is how we mock what would otherwise be post requests to a server. """ - command_responses = {} # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K + command_responses = ( + {} + ) # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K unknown_command_response = """[ { "result": 2, @@ -166,7 +171,9 @@ def new_post_function(url, *args, data="{}", **kwargs): elif isinstance(data, bytes): data = data.decode("utf8") if not isinstance(data, str): - pytest.fail(f"data argument to the mocked requests.post() is of unknown type {type(data)}") + pytest.fail( + f"data argument to the mocked requests.post() is of unknown type {type(data)}" + ) try: data = json.loads(data) @@ -212,6 +219,7 @@ def add_command_response(command_name, text): return add_command_response + @pytest.fixture def success_response(): return '''[ @@ -221,6 +229,7 @@ def success_response(): {"result": 0} ]''' + @pytest.fixture def error_response(): return '''[ @@ -231,6 +240,7 @@ def error_response(): {"text": "b", "arguments": {"arg1": "val1"}, "service": "d"} ]''' + @pytest.fixture def invalid_json_response(): return '''[ @@ -241,12 +251,14 @@ def invalid_json_response(): {"text": "b", "arguments": {"arg1": "val1"}, "service": "d" ]''' + @pytest.fixture def large_response(): return ''' ''' + def send_dummy_query(command="command"): return send_query( query=KeaQuery(command, [], {}), @@ -254,10 +266,12 @@ def send_dummy_query(command="command"): port=80, ) + ################################################################################ # Testing the list[KeaResponse] returned by send_query() # ################################################################################ + def test_success_responses_does_succeed(success_response, enqueue_post_response): enqueue_post_response("command", success_response) responses = send_dummy_query("command") @@ -268,6 +282,7 @@ def test_success_responses_does_succeed(success_response, enqueue_post_response) assert isinstance(response.arguments, dict) assert isinstance(response.service, str) + def test_error_responses_does_not_succeed(error_response, enqueue_post_response): enqueue_post_response("command", error_response) responses = send_dummy_query("command") @@ -278,7 +293,10 @@ def test_error_responses_does_not_succeed(error_response, enqueue_post_response) assert isinstance(response.arguments, dict) assert isinstance(response.service, str) -def test_invalid_json_responses_raises_jsonerror(invalid_json_response, enqueue_post_response): + +def test_invalid_json_responses_raises_jsonerror( + invalid_json_response, enqueue_post_response +): enqueue_post_response("command", invalid_json_response) with pytest.raises(JSONDecodeError): responses = send_dummy_query("command") @@ -288,6 +306,7 @@ def test_invalid_json_responses_raises_jsonerror(invalid_json_response, enqueue_ # Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from json # ################################################################################ + def test_correct_subnet_from_dhcp4_config_json(dhcp4_config): j = json.loads(dhcp4_config) subnet = KeaDhcpSubnet.from_json(j["Dhcp4"]["subnet4"][0]) @@ -297,6 +316,7 @@ def test_correct_subnet_from_dhcp4_config_json(dhcp4_config): assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) + def test_correct_config_from_dhcp4_config_json(dhcp4_config): j = json.loads(dhcp4_config) config = KeaDhcpConfig.from_json(j) @@ -310,7 +330,10 @@ def test_correct_config_from_dhcp4_config_json(dhcp4_config): assert config.dhcp_version == 4 assert config.config_hash is None -def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_shared_networks): + +def test_correct_config_from_dhcp4_config_w_shared_networks_json( + dhcp4_config_w_shared_networks, +): j = json.loads(dhcp4_config_w_shared_networks) config = KeaDhcpConfig.from_json(j) assert len(config.subnets) == 4 @@ -349,7 +372,7 @@ def test_correct_config_from_dhcp4_config_w_shared_networks_json(dhcp4_config_w_ assert config.config_hash is None -def dhcp4_config_response(dhcp4_config): +def response_json(dhcp4_config): return f''' [ {{ @@ -359,27 +382,36 @@ def dhcp4_config_response(dhcp4_config): ] ''' + ################################################################################ # Now we assume KeaDhcpSubnet and KeaDhcpConfig instantiation from json is # # correct. # # Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from server responses # ################################################################################ + def test_fetch_and_set_dhcp_config(dhcp4_config, enqueue_post_response): - enqueue_post_response("config-get", dhcp4_config_response(dhcp4_config)) + enqueue_post_response("config-get", response_json(dhcp4_config)) source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) assert source.kea_dhcp_config is None config = source.fetch_and_set_dhcp_config() - actual_config = KeaDhcpConfig.from_json(json.loads(dhcp4_config_response(dhcp4_config))[0]["arguments"]) + actual_config = KeaDhcpConfig.from_json( + json.loads(response_json(dhcp4_config))[0]["arguments"] + ) assert config == actual_config assert source.kea_dhcp_config == actual_config -def test_fetch_and_set_dhcp_config_w_shared_networks(dhcp4_config_w_shared_networks, enqueue_post_response): - enqueue_post_response("config-get", dhcp4_config_response(dhcp4_config_w_shared_networks)) + +def test_fetch_and_set_dhcp_config_w_shared_networks( + dhcp4_config_w_shared_networks, enqueue_post_response +): + enqueue_post_response("config-get", response_json(dhcp4_config_w_shared_networks)) source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) assert source.kea_dhcp_config is None config = source.fetch_and_set_dhcp_config() - actual_config = KeaDhcpConfig.from_json(json.loads(dhcp4_config_response(dhcp4_config_w_shared_networks))[0]["arguments"]) + actual_config = KeaDhcpConfig.from_json( + json.loads(response_json(dhcp4_config_w_shared_networks))[0]["arguments"] + ) assert config == actual_config assert source.kea_dhcp_config == actual_config From 14fe7d7d4b8f9c323533cd9975559af096da5b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 3 Jul 2024 15:32:34 +0200 Subject: [PATCH 31/77] Move helper functions and helper dataclasses down ... and make the public functions of the file come first --- python/nav/dhcp/kea_metrics.py | 598 +++++++++++++++++---------------- 1 file changed, 303 insertions(+), 295 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index b049120153..af6f0b45a7 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -16,6 +16,7 @@ >= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). """ +from __future__ import annotations import calendar import json import logging @@ -32,140 +33,180 @@ logger = logging.getLogger(__name__) -class KeaError(GeneralException): - """Error related to interaction with a Kea Control Agent""" - - -class KeaStatus(IntEnum): - """Status of a response sent from a Kea Control Agent.""" - - # Successful operation. - SUCCESS = 0 - # General failure. - ERROR = 1 - # Command is not supported. - UNSUPPORTED = 2 - # Successful operation, but failed to produce any results. - EMPTY = 3 - # Unsuccessful operation due to a conflict between the command arguments and the server state. - CONFLICT = 4 - - -@dataclass -class KeaResponse: +class KeaDhcpMetricSource(DhcpMetricSource): """ - Class representing the response to a REST query sent to a Kea - Control Agent. + Using `send_query()`, this class: + * Maintains an up-to-date `KeaDhcpConfig` representation of the + configuration of the Kea DHCP server with ip version + `self.dhcp_version` reachable via the Kea Control Agent listening + to port `self.rest_port` on IP addresses `self.rest_address` + * Queries the Kea Control Agent for statistics about each subnet + found in the `KeaDhcpConfig` representation and creates an + iterable of `DhcpMetric` that its superclass uses to fill a + graphite server with metrics. """ - result: int - text: str - arguments: dict - service: str - - @property - def success(self) -> bool: - return self.result == KeaStatus.SUCCESS + rest_address: str # IP address of the Kea Control Agent server + rest_port: int # Port of the Kea Control Agent server + rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. + dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server + kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. -@dataclass -class KeaQuery: - """Class representing a REST query to be sent to a Kea Control Agent.""" + def __init__( + self, + address: str, + port: int, + *args, + https: bool = True, + dhcp_version: int = 4, + **kwargs, + ): + super(*args, **kwargs) + self.rest_address = address + self.rest_port = port + self.rest_https = https + self.dchp_version = dhcp_version + self.kea_dhcp_config = None - command: str - arguments: dict + def fetch_metrics(self) -> list[DhcpMetric]: + """ + Implementation of the superclass method for fetching + standardised dhcp metrics. This method is used by the + superclass to feed data into a graphite server. + """ + metrics = [] + try: + s = requests.Session() + self.fetch_and_set_dhcp_config(s) + for subnet in self.kea_dhcp_config.subnets: + for statistic_key, metric_key in ( + ("total-addresses", DhcpMetricKey.MAX), + ("assigned-addresses", DhcpMetricKey.CUR), + ("declined-addresses", DhcpMetricKey.TOUCH), + ): # `statistic_key` is the name of the statistic used by Kea. `metric_key` is the name of the statistic used by NAV. + kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}" + query = KeaQuery( + command="statistic-get", + service=[f"dhcp{self.dchp_version}"], + arguments={ + "name": kea_statistic_name, + }, + ) + response = unwrap( + send_query( + query, + self.rest_address, + self.rest_port, + self.rest_https, + session=s, + ) + ) - # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. - service: list[str] + datapoints = response["arguments"].get(kea_statistic_name, []) + for value, timestamp in datapoints: + epochseconds = calendar.timegm( + time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + ) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! + metrics.append( + DhcpMetric(epochseconds, subnet.prefix, metric_key, value) + ) + used_config = self.kea_dhcp_config + self.fetch_and_set_dhcp_config(s) + if sorted(used_config.subnets) != sorted(self.kea_dhcp_config.subnets): + logger.warning( + "Subnet configuration was modified during metric fetching, " + "this may cause metric data being associated with wrong " + "subnet." + ) + except Timeout as err: + logger.warning( + "Connection to Kea Control Agent timed before or during metric " + "fetching. Some metrics may be missing.", + exc_info=err, + ) + except Exception as err: + # More detailed information should be logged by deeper exception handlers at the logging.DEBUG level. + logger.warning( + "Exception while fetching metrics from Kea Control Agent. Some " + "metrics may be missing.", + exc_info=err, + ) + finally: + s.close() -def send_query( - query: KeaQuery, - address: str, - port: int = 443, - https: bool = True, - session: requests.Session = None, - timeout: int = 10, -) -> list[KeaResponse]: - """ - Send `query` to a Kea Control Agent listening to `port` on IP - address `address`, using either http or https + return metrics - :param https: If True, use https. Otherwise, use http. + def fetch_and_set_dhcp_config(self, session=None): + """ + Fetch the current config used by the Kea DHCP server that + manages addresses of IP version `self.dhcp_version` from the Kea + Control Agent listening to `self.rest_port` on + `self.rest_address`. + """ + # Check if self.kea_dhcp_config is up to date + if not ( + self.kea_dhcp_config is None + or self.kea_dhcp_config.config_hash is None + or self.fetch_dhcp_config_hash(session=session) + != self.kea_dhcp_config.config_hash + ): + return self.kea_dhcp_config - :param session: Optional requests.Session to be used when sending - the query. Assumed to not be closed. session is not closed after - the end of this call, so that session can be used for persistent - http connections among different send_query calls. - """ - scheme = "https" if https else "http" - location = f"{scheme}://{address}:{port}/" - logger.debug("send_query: sending request to %s with query %r", location, query) - try: - if session is None: - r = requests.post( - location, - data=json.dumps(asdict(query)), - headers={"Content-Type": "application/json"}, - timeout=timeout, - ) - else: - r = session.post( - location, - data=json.dumps(asdict(query)), - headers={"Content-Type": "application/json"}, - timeout=timeout, + # self.kea_dhcp_config is not up to date, fetch new + query = KeaQuery( + command="config-get", + service=[f"dhcp{self.dchp_version}"], + arguments={}, + ) + response = unwrap( + send_query( + query, + self.rest_address, + self.rest_port, + self.rest_https, + session=session, ) - except HTTPError as err: - logger.debug( - "send_query: request to %s yielded an error: %d %s", - err.request.url, - err.response.status_code, - err.response.reason, ) - raise err - except Timeout as err: - logger.debug("send_query: request to %s timed out", err.request.url) - raise err - try: - response_json = r.json() - except JSONDecodeError as err: - logger.debug("send_query: expected json from %s, got %s", address, r.text) - raise err + self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) + return self.kea_dhcp_config - if isinstance(response_json, dict): - logger.debug( - "send_query: expected a json list of objects from %s, got %r", - address, - response_json, + def fetch_dhcp_config_hash(self, session=None): + """ + For Kea versions >= 2.4.0, fetch and return a hash of the + current configuration used by the Kea DHCP server. For Kea + versions < 2.4.0, return None. + """ + query = KeaQuery( + command="config-hash-get", + service=[f"dhcp{self.dchp_version}"], + arguments={}, ) - raise KeaError(f"bad response from {address}: {response_json!r}") - - responses = [] - for obj in response_json: - response = KeaResponse( - obj.get("result", KeaStatus.ERROR), - obj.get("text", ""), - obj.get("arguments", {}), - obj.get("service", ""), + response = unwrap( + send_query( + query, + self.rest_address, + self.rest_port, + self.rest_https, + session=session, + ), + require_success=False, ) - responses.append(response) - return responses - -def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: - """ - Helper function implementing the sequence of operations often done - on the list of responses returned by `send_query()` - """ - if len(responses) != 1: - raise KeaError("Received invalid amount of responses") - - response = responses[0] - if require_success and not response.success: - raise KeaError("Did not receive a successful response") - return response + if response.result == KeaStatus.UNSUPPORTED: + logger.debug( + "Kea DHCP%d server does not support quering for the hash of its config", + self.dchp_version, + ) + return None + elif response.success: + return response.arguments.get("hash", None) + else: + raise KeaError( + "Unexpected error while querying the hash of config file from DHCP server" + ) @dataclass @@ -285,201 +326,168 @@ def from_json( if len(config_json) > 1: raise KeaError("Did not expect len(configjson) > 1") - dhcp_version, config_json = config_json.popitem() - if dhcp_version == "Dhcp4": - dhcp_version = 4 - elif dhcp_version == "Dhcp6": - dhcp_version = 6 - else: - raise KeaError(f"Unsupported DHCP IP version '{dhcp_version}'") + dhcp_version, config_json = config_json.popitem() + if dhcp_version == "Dhcp4": + dhcp_version = 4 + elif dhcp_version == "Dhcp6": + dhcp_version = 6 + else: + raise KeaError(f"Unsupported DHCP IP version '{dhcp_version}'") + + subnets = [] + for obj in config_json.get(f"subnet{dhcp_version}", []): + subnet = KeaDhcpSubnet.from_json(obj) + subnets.append(subnet) + for obj in config_json.get("shared-networks", []): + for subobj in obj.get(f"subnet{dhcp_version}", []): + subnet = KeaDhcpSubnet.from_json(subobj) + subnets.append(subnet) + + return cls( + config_hash=config_hash, + dhcp_version=dhcp_version, + subnets=subnets, + ) + + +def send_query( + query: KeaQuery, + address: str, + port: int = 443, + https: bool = True, + session: requests.Session = None, + timeout: int = 10, +) -> list[KeaResponse]: + """ + Send `query` to a Kea Control Agent listening to `port` on IP + address `address`, using either http or https + + :param https: If True, use https. Otherwise, use http. + + :param session: Optional requests.Session to be used when sending + the query. Assumed to not be closed. session is not closed after + the end of this call, so that session can be used for persistent + http connections among different send_query calls. + """ + scheme = "https" if https else "http" + location = f"{scheme}://{address}:{port}/" + logger.debug("send_query: sending request to %s with query %r", location, query) + try: + if session is None: + r = requests.post( + location, + data=json.dumps(asdict(query)), + headers={"Content-Type": "application/json"}, + timeout=timeout, + ) + else: + r = session.post( + location, + data=json.dumps(asdict(query)), + headers={"Content-Type": "application/json"}, + timeout=timeout, + ) + except HTTPError as err: + logger.debug( + "send_query: request to %s yielded an error: %d %s", + err.request.url, + err.response.status_code, + err.response.reason, + exc_info=err, + ) + raise err + except Timeout as err: + logger.debug( + "send_query: request to %s timed out", err.request.url, exc_info=err + ) + raise err + + try: + response_json = r.json() + except JSONDecodeError as err: + logger.debug( + "send_query: expected json from %s, got %s", address, r.text, exc_info=err + ) + raise err - subnets = [] - for obj in config_json.get(f"subnet{dhcp_version}", []): - subnet = KeaDhcpSubnet.from_json(obj) - subnets.append(subnet) - for obj in config_json.get("shared-networks", []): - for subobj in obj.get(f"subnet{dhcp_version}", []): - subnet = KeaDhcpSubnet.from_json(subobj) - subnets.append(subnet) + if isinstance(response_json, dict): + err = KeaError(f"bad response from {address}: {response_json!r}") + logger.debug( + "send_query: expected a json list of objects from %s, got %r", + address, + response_json, + exc_info=err, + ) + raise err - return cls( - config_hash=config_hash, - dhcp_version=dhcp_version, - subnets=subnets, + responses = [] + for obj in response_json: + response = KeaResponse( + obj.get("result", KeaStatus.ERROR), + obj.get("text", ""), + obj.get("arguments", {}), + obj.get("service", ""), ) + responses.append(response) + return responses -class KeaDhcpMetricSource(DhcpMetricSource): +def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: """ - Using `send_query()`, this class: - * Maintains an up-to-date `KeaDhcpConfig` representation of the - configuration of the Kea DHCP server with ip version - `self.dhcp_version` reachable via the Kea Control Agent listening - to port `self.rest_port` on IP addresses `self.rest_address` - * Queries the Kea Control Agent for statistics about each subnet - found in the `KeaDhcpConfig` representation and creates an - iterable of `DhcpMetric` that its superclass uses to fill a - graphite server with metrics. + Helper function implementing the sequence of operations often done + on the list of responses returned by `send_query()` """ + if len(responses) != 1: + raise KeaError("Received invalid amount of responses") - rest_address: str # IP address of the Kea Control Agent server - rest_port: int # Port of the Kea Control Agent server - rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. + response = responses[0] + if require_success and not response.success: + raise KeaError("Did not receive a successful response") + return response - dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server - kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. - def __init__( - self, - address: str, - port: int, - *args, - https: bool = True, - dhcp_version: int = 4, - **kwargs, - ): - super(*args, **kwargs) - self.rest_address = address - self.rest_port = port - self.rest_https = https - self.dchp_version = dhcp_version - self.kea_dhcp_config = None +class KeaError(GeneralException): + """Error related to interaction with a Kea Control Agent""" - def fetch_and_set_dhcp_config(self, session=None): - """ - Fetch the current config used by the Kea DHCP server that - manages addresses of IP version `self.dhcp_version` from the Kea - Control Agent listening to `self.rest_port` on - `self.rest_address`. - """ - # Check if self.kea_dhcp_config is up to date - if not ( - self.kea_dhcp_config is None - or self.kea_dhcp_config.config_hash is None - or self.fetch_dhcp_config_hash(session=session) - != self.kea_dhcp_config.config_hash - ): - return self.kea_dhcp_config - # self.kea_dhcp_config is not up to date, fetch new - query = KeaQuery( - command="config-get", - service=[f"dhcp{self.dchp_version}"], - arguments={}, - ) - response = unwrap( - send_query( - query, - self.rest_address, - self.rest_port, - self.rest_https, - session=session, - ) - ) +class KeaStatus(IntEnum): + """Status of a response sent from a Kea Control Agent.""" - self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) - return self.kea_dhcp_config + # Successful operation. + SUCCESS = 0 + # General failure. + ERROR = 1 + # Command is not supported. + UNSUPPORTED = 2 + # Successful operation, but failed to produce any results. + EMPTY = 3 + # Unsuccessful operation due to a conflict between the command arguments and the server state. + CONFLICT = 4 - def fetch_dhcp_config_hash(self, session=None): - """ - For Kea versions >= 2.4.0, fetch and return a hash of the - current configuration used by the Kea DHCP server. For Kea - versions < 2.4.0, return None. - """ - query = KeaQuery( - command="config-hash-get", - service=[f"dhcp{self.dchp_version}"], - arguments={}, - ) - response = unwrap( - send_query( - query, - self.rest_address, - self.rest_port, - self.rest_https, - session=session, - ), - require_success=False, - ) - if response.result == KeaStatus.UNSUPPORTED: - logger.info( - "Kea DHCP%d server does not support quering for the hash of its config", - self.dchp_version, - ) - return None - elif response.success: - return response.arguments.get("hash", None) - else: - raise KeaError( - "Unexpected error while querying the hash of config file from DHCP server" - ) +@dataclass +class KeaResponse: + """ + Class representing the response to a REST query sent to a Kea + Control Agent. + """ - def fetch_metrics(self) -> list[DhcpMetric]: - """ - Implementation of the superclass method for fetching - standardised dhcp metrics. This method is used by the - superclass to feed data into a graphite server. - """ - metrics = [] - try: - s = requests.Session() - self.fetch_and_set_dhcp_config(s) - for subnet in self.kea_dhcp_config.subnets: - for statistic_key, metric_key in ( - ("total-addresses", DhcpMetricKey.MAX), - ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH), - ): # `statistic_key` is the name of the statistic used by Kea. `metric_key` is the name of the statistic used by NAV. - kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}" - query = KeaQuery( - command="statistic-get", - service=[f"dhcp{self.dchp_version}"], - arguments={ - "name": kea_statistic_name, - }, - ) - response = unwrap( - send_query( - query, - self.rest_address, - self.rest_port, - self.rest_https, - session=s, - ) - ) + result: int + text: str + arguments: dict + service: str - datapoints = response["arguments"].get(kea_statistic_name, []) - for value, timestamp in datapoints: - epochseconds = calendar.timegm( - time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") - ) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! - metrics.append( - DhcpMetric(epochseconds, subnet.prefix, metric_key, value) - ) + @property + def success(self) -> bool: + return self.result == KeaStatus.SUCCESS - used_config = self.kea_dhcp_config - self.fetch_and_set_dhcp_config(s) - if sorted(used_config.subnets) != sorted(self.kea_dhcp_config.subnets): - logger.warning( - "Subnet configuration was modified during metric fetching, " - "this may cause metric data being associated with wrong " - "subnet." - ) - except Timeout as err: - logger.warning( - "Connection to Kea Control Agent timed before or during metric " - "fetching. Some metrics may be missing.", - exc_info=err, - ) - except Exception as err: - # More detailed information should be logged by deeper exception handlers at the logging.DEBUG level. - logger.warning( - "Exception while fetching metrics from Kea Control Agent. Some " - "metrics may be missing.", - exc_info=err, - ) - finally: - s.close() - return metrics +@dataclass +class KeaQuery: + """Class representing a REST query to be sent to a Kea Control Agent.""" + + command: str + arguments: dict + + # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. + service: list[str] From f02b64a862ef4b59cab2766f40887cc95507418a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 3 Jul 2024 15:46:31 +0200 Subject: [PATCH 32/77] Streamline logging and exception handling What: All caught exceptions in helper/private functions should log about the specific exceptions if suitable, and then reraises the exception as a KeaError that any public-facing functions can catch in one scoop and handle as fits. --- python/nav/dhcp/kea_metrics.py | 278 ++++++++++++----------- tests/unittests/dhcp/kea_metrics_test.py | 80 ++++++- 2 files changed, 217 insertions(+), 141 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index af6f0b45a7..e006c0443c 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -4,7 +4,7 @@ RESTful-queries IPC nav <---------------> Kea Control Agent <=====> Kea DHCP4 server / Kea DHCP6 server - (json) + send_query() No additional hook libraries are assumed to be included with the Kea Control Agent that is queried, meaning this module will be able to gather DHCP @@ -15,7 +15,6 @@ * See also the Kea Control Agent documentation. This script assumes Kea versions >= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). """ - from __future__ import annotations import calendar import json @@ -30,7 +29,7 @@ from requests.exceptions import JSONDecodeError, HTTPError, Timeout from typing import Optional -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class KeaDhcpMetricSource(DhcpMetricSource): @@ -51,7 +50,7 @@ class KeaDhcpMetricSource(DhcpMetricSource): rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server - kea_dhcp_config: dict # The configuration, i.e. most static pieces of information, of the Kea DHCP server. + kea_dhcp_config: KeaDhcpConfig # The configuration, i.e. most static pieces of information, of the Kea DHCP server. def __init__( self, @@ -69,6 +68,7 @@ def __init__( self.dchp_version = dhcp_version self.kea_dhcp_config = None + def fetch_metrics(self) -> list[DhcpMetric]: """ Implementation of the superclass method for fetching @@ -80,12 +80,12 @@ def fetch_metrics(self) -> list[DhcpMetric]: s = requests.Session() self.fetch_and_set_dhcp_config(s) for subnet in self.kea_dhcp_config.subnets: - for statistic_key, metric_key in ( + for kea_key, nav_key in ( ("total-addresses", DhcpMetricKey.MAX), ("assigned-addresses", DhcpMetricKey.CUR), ("declined-addresses", DhcpMetricKey.TOUCH), - ): # `statistic_key` is the name of the statistic used by Kea. `metric_key` is the name of the statistic used by NAV. - kea_statistic_name = f"subnet[{subnet.id}].{statistic_key}" + ): + kea_statistic_name = f"subnet[{subnet.id}].{kea_key}" query = KeaQuery( command="statistic-get", service=[f"dhcp{self.dchp_version}"], @@ -105,31 +105,23 @@ def fetch_metrics(self) -> list[DhcpMetric]: datapoints = response["arguments"].get(kea_statistic_name, []) for value, timestamp in datapoints: - epochseconds = calendar.timegm( - time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") - ) # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! + # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! + epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) metrics.append( - DhcpMetric(epochseconds, subnet.prefix, metric_key, value) + DhcpMetric(epochseconds, subnet.prefix, nav_key, value) ) used_config = self.kea_dhcp_config self.fetch_and_set_dhcp_config(s) if sorted(used_config.subnets) != sorted(self.kea_dhcp_config.subnets): - logger.warning( + _logger.warning( "Subnet configuration was modified during metric fetching, " "this may cause metric data being associated with wrong " "subnet." ) - except Timeout as err: - logger.warning( - "Connection to Kea Control Agent timed before or during metric " - "fetching. Some metrics may be missing.", - exc_info=err, - ) - except Exception as err: - # More detailed information should be logged by deeper exception handlers at the logging.DEBUG level. - logger.warning( - "Exception while fetching metrics from Kea Control Agent. Some " + except KeaError as err: + _logger.warning( + "Error while fetching metrics from Kea Control Agent. Some " "metrics may be missing.", exc_info=err, ) @@ -138,7 +130,8 @@ def fetch_metrics(self) -> list[DhcpMetric]: return metrics - def fetch_and_set_dhcp_config(self, session=None): + + def fetch_and_set_dhcp_config(self, session=None) -> KeaDhcpConfig: """ Fetch the current config used by the Kea DHCP server that manages addresses of IP version `self.dhcp_version` from the Kea @@ -173,6 +166,7 @@ def fetch_and_set_dhcp_config(self, session=None): self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) return self.kea_dhcp_config + def fetch_dhcp_config_hash(self, session=None): """ For Kea versions >= 2.4.0, fetch and return a hash of the @@ -195,75 +189,19 @@ def fetch_dhcp_config_hash(self, session=None): require_success=False, ) - if response.result == KeaStatus.UNSUPPORTED: - logger.debug( - "Kea DHCP%d server does not support quering for the hash of its config", - self.dchp_version, - ) - return None - elif response.success: + if response.success: return response.arguments.get("hash", None) + elif response.result == KeaStatus.UNSUPPORTED: + _logger.debug("Kea server does not support command 'config-hash-get'") + return None else: - raise KeaError( - "Unexpected error while querying the hash of config file from DHCP server" + _logger.debug( + "Unexpected response (%s) after querying the hash of config file from Kea " + "DHCP%d server", + repr(response), + self.dhcp_version, ) - - -@dataclass -class KeaDhcpSubnet: - """Class representing information about a subnet managed by a Kea DHCP server.""" - - id: int # either specified in the server config or assigned automatically by the dhcp server - prefix: IP # e.g. 192.0.2.1/24 - pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] - - @classmethod - def from_json(cls, subnet_json: dict): - """ - Initialize and return a Subnet instance based on json - - :param json: python dictionary that is structured the same way as the - json object representing a subnet in the Kea DHCP config file. - Example: - { - "id": 0 - "subnet": "192.0.2.0/24", - "pools": [ - { - "pool": "192.0.2.1 - 192.0.2.100" - }, - { - "pool": "192.0.2.128/26" - } - ] - } - """ - if "id" not in subnet_json: - raise KeaError("Expected subnetjson['id'] to exist") - id = subnet_json["id"] - - if "subnet" not in subnet_json: - raise KeaError("Expected subnetjson['subnet'] to exist") - prefix = IP(subnet_json["subnet"]) - - pools = [] - for obj in subnet_json.get("pools", []): - pool = obj["pool"] - if "-" in pool: # TODO: Error checking? - # pool == "x.x.x.x - y.y.y.y" - start, end = (IP(ip) for ip in pool.split("-")) - else: - # pool == "x.x.x.x/nn" - pool = IP(pool) - start, end = pool[0], pool[-1] - pools.append((start, end)) - - return cls( - id=id, - prefix=prefix, - pools=pools, - ) - + return None @dataclass class KeaDhcpConfig: @@ -323,16 +261,24 @@ def from_json( :param hash: hash of the Kea DHCP config file as returned by a `config-hash-get` query on the kea-ctrl-agent REST server. """ - if len(config_json) > 1: - raise KeaError("Did not expect len(configjson) > 1") + if len(config_json) != 1: + _logger.debug( + "KeaDhcpConfig.from_json: expected outermost object to have one key, got: %r", + config_json + ) + raise KeaError("Invalid DHCP config JSON") - dhcp_version, config_json = config_json.popitem() - if dhcp_version == "Dhcp4": + service, config_json = config_json.popitem() + if service == "Dhcp4": dhcp_version = 4 - elif dhcp_version == "Dhcp6": + elif service == "Dhcp6": dhcp_version = 6 else: - raise KeaError(f"Unsupported DHCP IP version '{dhcp_version}'") + _logger.debug( + "KeaDhcpConfig.from_json: config JSON from unknown Kea service: %s", + service + ) + raise KeaError(f"Unsupported Kea service '{service}'") subnets = [] for obj in config_json.get(f"subnet{dhcp_version}", []): @@ -350,6 +296,70 @@ def from_json( ) +@dataclass +class KeaDhcpSubnet: + """Class representing information about a subnet managed by a Kea DHCP server.""" + + id: int # either specified in the server config or assigned automatically by the dhcp server + prefix: IP # e.g. 192.0.2.1/24 + pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] + + @classmethod + def from_json(cls, subnet_json: dict): + """ + Initialize and return a Subnet instance based on json + + :param json: python dictionary that is structured the same way as the + json object representing a subnet in the Kea DHCP config file. + Example: + { + "id": 0 + "subnet": "192.0.2.0/24", + "pools": [ + { + "pool": "192.0.2.1 - 192.0.2.100" + }, + { + "pool": "192.0.2.128/26" + } + ] + } + """ + if "id" not in subnet_json: + _logger.debug( + "KeaDhcpSubnet.from_json: subnet JSON missing key 'id': %r", + subnet_json + ) + raise KeaError("Expected subnetjson['id'] to exist") + id = subnet_json["id"] + + if "subnet" not in subnet_json: + _logger.debug( + "KeaDhcpSubnet.from_json: subnet JSON missing key 'subnet': %r", + subnet_json + ) + raise KeaError("Expected subnetjson['subnet'] to exist") + prefix = IP(subnet_json["subnet"]) + + pools = [] + for obj in subnet_json.get("pools", []): + pool = obj["pool"] + if "-" in pool: # TODO: Error checking? + # pool == "x.x.x.x - y.y.y.y" + start, end = (IP(ip) for ip in pool.split("-")) + else: + # pool == "x.x.x.x/nn" + pool = IP(pool) + start, end = pool[0], pool[-1] + pools.append((start, end)) + + return cls( + id=id, + prefix=prefix, + pools=pools, + ) + + def send_query( query: KeaQuery, address: str, @@ -371,7 +381,7 @@ def send_query( """ scheme = "https" if https else "http" location = f"{scheme}://{address}:{port}/" - logger.debug("send_query: sending request to %s with query %r", location, query) + _logger.debug("send_query: sending request to %s with query %r", location, query) try: if session is None: r = requests.post( @@ -387,38 +397,31 @@ def send_query( headers={"Content-Type": "application/json"}, timeout=timeout, ) - except HTTPError as err: - logger.debug( - "send_query: request to %s yielded an error: %d %s", - err.request.url, - err.response.status_code, - err.response.reason, - exc_info=err, - ) - raise err - except Timeout as err: - logger.debug( - "send_query: request to %s timed out", err.request.url, exc_info=err + except RequestException as err: + _logger.debug( + "send_query: Requests failed to complete request to %s with query %r", + location, + query, ) - raise err + raise KeaError() from err try: response_json = r.json() except JSONDecodeError as err: - logger.debug( - "send_query: expected json from %s, got %s", address, r.text, exc_info=err + _logger.debug( + "send_query: invalid json from %s, got: %s", + address, + r.text, ) - raise err + raise KeaError() from err if isinstance(response_json, dict): - err = KeaError(f"bad response from {address}: {response_json!r}") - logger.debug( - "send_query: expected a json list of objects from %s, got %r", + _logger.debug( + "send_query: expected a json list of objects from %s, got: %r", address, response_json, - exc_info=err, ) - raise err + raise KeaError(f"bad response from {address}: {response_json!r}") responses = [] for obj in response_json: @@ -438,33 +441,16 @@ def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: on the list of responses returned by `send_query()` """ if len(responses) != 1: + _logger.debug("unwrap: received %d responses, expected 1 (%r)", responses) raise KeaError("Received invalid amount of responses") response = responses[0] if require_success and not response.success: + _logger.debug("unwrap: received an unsuccessful response (%r)", response) raise KeaError("Did not receive a successful response") return response -class KeaError(GeneralException): - """Error related to interaction with a Kea Control Agent""" - - -class KeaStatus(IntEnum): - """Status of a response sent from a Kea Control Agent.""" - - # Successful operation. - SUCCESS = 0 - # General failure. - ERROR = 1 - # Command is not supported. - UNSUPPORTED = 2 - # Successful operation, but failed to produce any results. - EMPTY = 3 - # Unsuccessful operation due to a conflict between the command arguments and the server state. - CONFLICT = 4 - - @dataclass class KeaResponse: """ @@ -481,7 +467,6 @@ class KeaResponse: def success(self) -> bool: return self.result == KeaStatus.SUCCESS - @dataclass class KeaQuery: """Class representing a REST query to be sent to a Kea Control Agent.""" @@ -491,3 +476,22 @@ class KeaQuery: # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. service: list[str] + +class KeaError(GeneralException): + """Error related to interaction with a Kea Control Agent""" + +class KeaStatus(IntEnum): + """Status of a response sent from a Kea Control Agent.""" + + # Successful operation. + SUCCESS = 0 + # General failure. + ERROR = 1 + # Command is not supported. + UNSUPPORTED = 2 + # Successful operation, but failed to produce any results. + EMPTY = 3 + # Unsuccessful operation due to a conflict between the command arguments and the server state. + CONFLICT = 4 + + diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 6bfc291f0f..8e3952f13d 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -4,9 +4,47 @@ import requests from IPy import IP import json +import logging +import re from requests.exceptions import JSONDecodeError +class LogChecker: + def __init__(self, caplog): + self.caplog = caplog + + def clear(self): + self.caplog.clear() + + def has_entries(self, level, exception=None, regex=None, n=None): + """ + Check if there is any log entries of logging level `level`, optionally + made for an exception `exception`, optionally with message fully + matching regex `regex`, and optionally requiring that there is exactly + `n` such records logged. + """ + def causes(e: BaseException): + while e: + yield type(e) + e = e.__cause__ + + entries = [entry for entry in self.caplog.records + if entry.levelno == level + and (exception is None + and entry.exc_info is None + or entry.exc_info is not None + and exception in causes(entry.exc_info[1])) + and (regex is None or re.fullmatch(regex, entry.message.lower(), re.DOTALL))] + return n is None and len(entries) > 0 or len(entries) == n + + +@pytest.fixture +def testlog(caplog): + caplog.clear() + caplog.set_level(logging.DEBUG) + return LogChecker(caplog) + + @pytest.fixture def dhcp4_config(): return ''' @@ -293,14 +331,48 @@ def test_error_responses_does_not_succeed(error_response, enqueue_post_response) assert isinstance(response.arguments, dict) assert isinstance(response.service, str) +################################################################################ +# Testing correct error handling if Kea server returns invalid JSON # +################################################################################ -def test_invalid_json_responses_raises_jsonerror( - invalid_json_response, enqueue_post_response +def test_invalid_json_response( + testlog, invalid_json_response, enqueue_post_response ): enqueue_post_response("command", invalid_json_response) - with pytest.raises(JSONDecodeError): + testlog.clear() + with pytest.raises(KeaError): responses = send_dummy_query("command") - + assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") + + enqueue_post_response("config-get", lambda **_: invalid_json_response) + enqueue_post_response("statistic-get", lambda **_: invalid_json_response) + testlog.clear() + source = KeaDhcpMetricSource(address="192.0.2.1", port=80) + with pytest.raises(KeaError): + source.fetch_and_set_dhcp_config() + assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") + + # fetch_dhcp_config_hash should not raise when the server does not support + # config-hash-get command + testlog.clear() + h = source.fetch_dhcp_config_hash() + assert h == None + assert testlog.has_entries(logging.DEBUG, regex=".*no.*support.*hash.*|.*hash.*no.*support.*") + + # fetch_dhcp_config_hash should raise when the server returns invalid + # json + enqueue_post_response("config-hash-get", lambda **_: invalid_json_response) + testlog.clear() + with pytest.raises(KeaError): + source.fetch_and_set_dhcp_config() + assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") + + # FUNCTIONS USED EXTERNALLY SHOULD CATCH EXCEPTIONS AND LOG WARNINGS + # fetch_metrics is a method also used external to the module, and thus + # instead of raising it should log when the server returns invalid json + testlog.clear() + source.fetch_metrics() + assert testlog.has_entries(logging.WARNING, JSONDecodeError, n=1) ################################################################################ # Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from json # From 9456aa89baecfec89e4646af5498b72853c02d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 3 Jul 2024 15:50:31 +0200 Subject: [PATCH 33/77] Add dhcp6 config test --- python/nav/dhcp/kea_metrics.py | 22 ++--- tests/unittests/dhcp/kea_metrics_test.py | 105 +++++++++++++++++++---- 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index e006c0443c..7911d3acd7 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -68,7 +68,6 @@ def __init__( self.dchp_version = dhcp_version self.kea_dhcp_config = None - def fetch_metrics(self) -> list[DhcpMetric]: """ Implementation of the superclass method for fetching @@ -106,7 +105,9 @@ def fetch_metrics(self) -> list[DhcpMetric]: datapoints = response["arguments"].get(kea_statistic_name, []) for value, timestamp in datapoints: # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! - epochseconds = calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) + epochseconds = calendar.timegm( + time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + ) metrics.append( DhcpMetric(epochseconds, subnet.prefix, nav_key, value) ) @@ -130,7 +131,6 @@ def fetch_metrics(self) -> list[DhcpMetric]: return metrics - def fetch_and_set_dhcp_config(self, session=None) -> KeaDhcpConfig: """ Fetch the current config used by the Kea DHCP server that @@ -166,7 +166,6 @@ def fetch_and_set_dhcp_config(self, session=None) -> KeaDhcpConfig: self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) return self.kea_dhcp_config - def fetch_dhcp_config_hash(self, session=None): """ For Kea versions >= 2.4.0, fetch and return a hash of the @@ -203,6 +202,7 @@ def fetch_dhcp_config_hash(self, session=None): ) return None + @dataclass class KeaDhcpConfig: """ @@ -264,7 +264,7 @@ def from_json( if len(config_json) != 1: _logger.debug( "KeaDhcpConfig.from_json: expected outermost object to have one key, got: %r", - config_json + config_json, ) raise KeaError("Invalid DHCP config JSON") @@ -276,7 +276,7 @@ def from_json( else: _logger.debug( "KeaDhcpConfig.from_json: config JSON from unknown Kea service: %s", - service + service, ) raise KeaError(f"Unsupported Kea service '{service}'") @@ -327,8 +327,7 @@ def from_json(cls, subnet_json: dict): """ if "id" not in subnet_json: _logger.debug( - "KeaDhcpSubnet.from_json: subnet JSON missing key 'id': %r", - subnet_json + "KeaDhcpSubnet.from_json: subnet JSON missing key 'id': %r", subnet_json ) raise KeaError("Expected subnetjson['id'] to exist") id = subnet_json["id"] @@ -336,7 +335,7 @@ def from_json(cls, subnet_json: dict): if "subnet" not in subnet_json: _logger.debug( "KeaDhcpSubnet.from_json: subnet JSON missing key 'subnet': %r", - subnet_json + subnet_json, ) raise KeaError("Expected subnetjson['subnet'] to exist") prefix = IP(subnet_json["subnet"]) @@ -467,6 +466,7 @@ class KeaResponse: def success(self) -> bool: return self.result == KeaStatus.SUCCESS + @dataclass class KeaQuery: """Class representing a REST query to be sent to a Kea Control Agent.""" @@ -477,9 +477,11 @@ class KeaQuery: # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. service: list[str] + class KeaError(GeneralException): """Error related to interaction with a Kea Control Agent""" + class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" @@ -493,5 +495,3 @@ class KeaStatus(IntEnum): EMPTY = 3 # Unsuccessful operation due to a conflict between the command arguments and the server state. CONFLICT = 4 - - diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 8e3952f13d..7e23eca865 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -23,18 +23,24 @@ def has_entries(self, level, exception=None, regex=None, n=None): matching regex `regex`, and optionally requiring that there is exactly `n` such records logged. """ + def causes(e: BaseException): while e: yield type(e) e = e.__cause__ - entries = [entry for entry in self.caplog.records - if entry.levelno == level - and (exception is None - and entry.exc_info is None - or entry.exc_info is not None - and exception in causes(entry.exc_info[1])) - and (regex is None or re.fullmatch(regex, entry.message.lower(), re.DOTALL))] + entries = [ + entry + for entry in self.caplog.records + if entry.levelno == level + and ( + exception is None + and entry.exc_info is None + or entry.exc_info is not None + and exception in causes(entry.exc_info[1]) + ) + and (regex is None or re.fullmatch(regex, entry.message.lower(), re.DOTALL)) + ] return n is None and len(entries) > 0 or len(entries) == n @@ -45,6 +51,51 @@ def testlog(caplog): return LogChecker(caplog) +@pytest.fixture +def dhcp6_config(): + return '''{ +"Dhcp6": { + "valid-lifetime": 4000, + "renew-timer": 1000, + "rebind-timer": 2000, + "preferred-lifetime": 3000, + + "interfaces-config": { + "interfaces": [ "eth0" ] + }, + + "lease-database": { + "type": "memfile", + "persist": true, + "name": "/var/lib/kea/dhcp6.leases" + }, + + "subnet6": [ + { + "id": 1, + "subnet": "2001:db8:1:1::/64", + "pools": [ + { + "pool": "2001:db8:1:1::1-2001:db8:1:1::ffff" + } + ] + }, + { + "id": 2, + "subnet": "2001:db8:1:2::/64", + "pools": [ + { + "pool": "2001:db8:1:2::1-2001:db8:1:2::ffff" + }, + { + "pool": "2001:db8:1:2::1:0/112" + } + ] + } + ] +} +}''' + @pytest.fixture def dhcp4_config(): return ''' @@ -197,10 +248,10 @@ def enqueue_post_response(monkeypatch): {} ) # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K unknown_command_response = """[ - { + {{ "result": 2, "text": "'{0}' command not supported." - } + }} ]""" def new_post_function(url, *args, data="{}", **kwargs): @@ -227,7 +278,7 @@ def new_post_function(url, *args, data="{}", **kwargs): if fifo: first = fifo[0] if callable(first): - text = first() + text = first(arguments=data.get("arguments", {}), service=data.get("service", [])) else: text = str(first) fifo.popleft() @@ -331,13 +382,13 @@ def test_error_responses_does_not_succeed(error_response, enqueue_post_response) assert isinstance(response.arguments, dict) assert isinstance(response.service, str) + ################################################################################ # Testing correct error handling if Kea server returns invalid JSON # ################################################################################ -def test_invalid_json_response( - testlog, invalid_json_response, enqueue_post_response -): + +def test_invalid_json_response(testlog, invalid_json_response, enqueue_post_response): enqueue_post_response("command", invalid_json_response) testlog.clear() with pytest.raises(KeaError): @@ -357,7 +408,9 @@ def test_invalid_json_response( testlog.clear() h = source.fetch_dhcp_config_hash() assert h == None - assert testlog.has_entries(logging.DEBUG, regex=".*no.*support.*hash.*|.*hash.*no.*support.*") + assert testlog.has_entries( + logging.DEBUG, regex=".*no.*support.*hash.*|.*hash.*no.*support.*" + ) # fetch_dhcp_config_hash should raise when the server returns invalid # json @@ -374,6 +427,7 @@ def test_invalid_json_response( source.fetch_metrics() assert testlog.has_entries(logging.WARNING, JSONDecodeError, n=1) + ################################################################################ # Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from json # ################################################################################ @@ -403,6 +457,28 @@ def test_correct_config_from_dhcp4_config_json(dhcp4_config): assert config.config_hash is None +def test_correct_config_from_dhcp6_config_json(dhcp6_config): + j = json.loads(dhcp6_config) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 2 + subnet1 = config.subnets[0] + assert subnet1.id == 1 + assert subnet1.prefix == IP("2001:db8:1:1::/64") + assert len(subnet1.pools) == 1 + assert subnet1.pools[0] == (IP("2001:db8:1:1::1"), IP("2001:db8:1:1::ffff")) + assert config.dhcp_version == 6 + assert config.config_hash is None + + subnet2 = config.subnets[1] + assert subnet2.id == 2 + assert subnet2.prefix == IP("2001:db8:1:2::/64") + assert len(subnet2.pools) == 2 + assert subnet2.pools[0] == (IP("2001:db8:1:2::1"), IP("2001:db8:1:2::ffff")) + assert subnet2.pools[1] == (IP("2001:db8:1:2::1:0"), IP("2001:db8:1:2::1:ffff")) + assert config.dhcp_version == 6 + assert config.config_hash is None + + def test_correct_config_from_dhcp4_config_w_shared_networks_json( dhcp4_config_w_shared_networks, ): @@ -443,7 +519,6 @@ def test_correct_config_from_dhcp4_config_w_shared_networks_json( assert config.dhcp_version == 4 assert config.config_hash is None - def response_json(dhcp4_config): return f''' [ From 95e18ad7395dc4bac0efa9c75c0c08718c1b6c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Thu, 4 Jul 2024 15:09:22 +0200 Subject: [PATCH 34/77] Add tests --- tests/unittests/dhcp/kea_metrics_test.py | 522 ++++++++++++----------- 1 file changed, 273 insertions(+), 249 deletions(-) diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 7e23eca865..d79f0f6655 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -6,7 +6,254 @@ import json import logging import re -from requests.exceptions import JSONDecodeError +from requests.exceptions import JSONDecodeError, HTTPError, Timeout + +class TestSendQuery: + """Testing the list[KeaResponse] returned by send_query()""" + def test_response_with_success_status_should_succeed_and_be_logged(self, success_response, enqueue_post_response, testlog): + enqueue_post_response("command", success_response) + testlog.clear() + responses = send_dummy_query("command") + assert len(responses) == 4 + for response in responses: + assert response.success + assert isinstance(response.text, str) + assert isinstance(response.arguments, dict) + assert isinstance(response.service, str) + assert testlog.has_entries(logging.DEBUG, regexes=("query", "sen(d|t)", "192.0.2.2:80")) + + def test_response_with_error_status_should_succeed_and_be_logged(self, error_response, enqueue_post_response, testlog): + enqueue_post_response("command", error_response) + responses = send_dummy_query("command") + assert len(responses) == 5 + for response in responses: + assert not response.success + assert isinstance(response.text, str) + assert isinstance(response.arguments, dict) + assert isinstance(response.service, str) + assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) + assert testlog.has_entries(logging.DEBUG, n=1) + + def test_exceptions_should_be_logged_and_reraised_as_KeaError(self, enqueue_post_response, testlog): + enqueue_post_response("httperror", raiser(HTTPError)) + enqueue_post_response("timeout", raiser(Timeout)) + query = KeaQuery("httperror", [], {}) + with pytest.raises(KeaError): + responses = send_query(query, "192.0.2.2", 80) + assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) + assert testlog.has_entries(logging.DEBUG, regexes=("query", "fail|error|exception", "192.0.2.2:80")) + + testlog.clear() + + query = KeaQuery("timeout", [], {}) + with pytest.raises(KeaError): + responses = send_query(query, "192.0.2.2", 80) + assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) + assert testlog.has_entries(logging.DEBUG, regexes=("[^_]query", "fail|error|exception", "192.0.2.2:80")) + + def test_response_with_invalid_json_should_raise_and_be_logged(self, invalid_json_response, enqueue_post_response, testlog): + enqueue_post_response("command", invalid_json_response) + testlog.clear() + with pytest.raises(KeaError): + responses = send_dummy_query("command") + assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) + assert testlog.has_entries(logging.DEBUG, regexes=("invalid", "json", "192.0.2.2:80")) + +class TestProcessingJsonIntoDataclass: + """ + Testing that json formatted Kea DHCP configuration is correctly processed + into python dataclasses + """ + def test_dhcp4_config_json_should_be_correctly_processed_into_KeaDhcpConfig( + self, + dhcp4_config, + dhcp4_config_with_shared_networks + ): + # Massaging and processing json with subnet4 list + j = json.loads(dhcp4_config) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 1 + subnet = config.subnets[0] + assert subnet.id == 1 + assert subnet.prefix == IP("192.0.0.0/8") + assert len(subnet.pools) == 2 + assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) + assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) + assert config.dhcp_version == 4 + assert config.config_hash is None + + # Massaging and processing json with shared_networks list AND subnet4 list + j = json.loads(dhcp4_config_with_shared_networks) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 4 + subnets = {subnet.id: subnet for subnet in config.subnets} + + subnet1 = subnets[1] + assert subnet1.id == 1 + assert subnet1.prefix == IP("192.0.1.0/24") + assert len(subnet1.pools) == 1 + assert subnet1.pools[0] == (IP("192.0.1.1"), IP("192.0.1.200")) + assert config.dhcp_version == 4 + assert config.config_hash is None + + subnet2 = subnets[2] + assert subnet2.id == 2 + assert subnet2.prefix == IP("192.0.2.0/24") + assert len(subnet2.pools) == 1 + assert subnet2.pools[0] == (IP("192.0.2.100"), IP("192.0.2.199")) + assert config.dhcp_version == 4 + assert config.config_hash is None + + subnet3 = subnets[3] + assert subnet3.id == 3 + assert subnet3.prefix == IP("192.0.3.0/24") + assert len(subnet3.pools) == 1 + assert subnet3.pools[0] == (IP("192.0.3.100"), IP("192.0.3.199")) + assert config.dhcp_version == 4 + assert config.config_hash is None + + subnet4 = subnets[4] + assert subnet4.id == 4 + assert subnet4.prefix == IP("10.0.0.0/8") + assert len(subnet4.pools) == 1 + assert subnet4.pools[0] == (IP("10.0.0.1"), IP("10.0.0.99")) + assert config.dhcp_version == 4 + assert config.config_hash is None + + def test_dhcp6_config_json_should_be_correctly_processed_into_KeaDhcpConfig(self, dhcp6_config): + j = json.loads(dhcp6_config) + config = KeaDhcpConfig.from_json(j) + assert len(config.subnets) == 2 + subnet1 = config.subnets[0] + assert subnet1.id == 1 + assert subnet1.prefix == IP("2001:db8:1:1::/64") + assert len(subnet1.pools) == 1 + assert subnet1.pools[0] == (IP("2001:db8:1:1::1"), IP("2001:db8:1:1::ffff")) + assert config.dhcp_version == 6 + assert config.config_hash is None + + subnet2 = config.subnets[1] + assert subnet2.id == 2 + assert subnet2.prefix == IP("2001:db8:1:2::/64") + assert len(subnet2.pools) == 2 + assert subnet2.pools[0] == (IP("2001:db8:1:2::1"), IP("2001:db8:1:2::ffff")) + assert subnet2.pools[1] == (IP("2001:db8:1:2::1:0"), IP("2001:db8:1:2::1:ffff")) + assert config.dhcp_version == 6 + assert config.config_hash is None + +""" +def test_invalid_json_response(testlog, invalid_json_response, enqueue_post_response): + enqueue_post_response("config-get", lambda **_: invalid_json_response) + enqueue_post_response("statistic-get", lambda **_: invalid_json_response) + testlog.clear() + source = KeaDhcpMetricSource(address="192.0.2.1", port=80) + with pytest.raises(KeaError): + source.fetch_and_set_dhcp_config() + assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") + + # fetch_dhcp_config_hash should not raise when the server does not support + # config-hash-get command + testlog.clear() + h = source.fetch_dhcp_config_hash() + assert h == None + assert testlog.has_entries( + logging.DEBUG, regex=".*no.*support.*hash.*|.*hash.*no.*support.*" + ) + + # fetch_dhcp_config_hash should raise when the server returns invalid + # json + enqueue_post_response("config-hash-get", lambda **_: invalid_json_response) + testlog.clear() + with pytest.raises(KeaError): + source.fetch_and_set_dhcp_config() + assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") + + # FUNCTIONS USED EXTERNALLY SHOULD CATCH EXCEPTIONS AND LOG WARNINGS + # fetch_metrics is a method also used external to the module, and thus + # instead of raising it should log when the server returns invalid json + testlog.clear() + source.fetch_metrics() + assert testlog.has_entries(logging.WARNING, JSONDecodeError, n=1) +""" + +class TestKeaDhcpMetricSource: + def test_when_no_cached_KeaDhcpConfig_exist_should_fetch_and_set_correct_KeaDhcpConfig( + self, + dhcp4_config, + dhcp4_config_with_shared_networks, + dhcp6_config, + enqueue_post_response, + testlog, + ): + for config_string in dhcp4_config, dhcp6_config, dhcp4_config_with_shared_networks: + testlog.clear() + enqueue_post_response("config-get", response_json(config_string)) + source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) + assert source.kea_dhcp_config is None + config = source.fetch_and_set_dhcp_config() + actual_config = KeaDhcpConfig.from_json( + json.loads(config_string) + ) + assert config == actual_config + assert source.kea_dhcp_config == actual_config + assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.1")) + + def test_when_cached_KeaDhcpConfig_exist_and_local_hash_match_with_server_hash_should_not_fetch_new_KeaDhcpConfig( + self, + dhcp4_config, + dhcp4_config_with_shared_networks, + enqueue_post_response, + ): + enqueue_post_response("config-get", response_json(dhcp4_config)) + source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) + source.fetch_and_set_dhcp_config() + enqueue_post_response("config-hash-get", response_json(f'{{hash: "{source.kea_dhcp_config.config_hash}"}}')) + # The command 'config-hash-get' is set to return the same hash as is already cached. Thus, the test should fail if 'config-get' is queried. + enqueue_post_response("config-get", lambda **_: pytest.fail()) + source.fetch_and_set_dhcp_config() + + def test_when_cached_KeaDhcpConfig_exist_and_local_hash_doesnt_match_with_server_hash_should_fetch_new_KeaDhcpConfig( + self, + dhcp4_config, + dhcp4_config_with_shared_networks, + enqueue_post_response, + ): + enqueue_post_response("config-get", response_json(dhcp4_config)) + source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) + source.fetch_and_set_dhcp_config() + assert source.kea_dhcp_config == KeaDhcpConfig.from_json(dhcp4_config) + + old_hash = source.kea_dhcp_config.config_hash + new_hash = "0" + old_hash[1:] if old_hash[0] != "0" else "1" + old_hash[1:] + + enqueue_post_response("config-hash-get", response_json(f'{{hash: "{new_hash}"}}')) + enqueue_post_response("config-get", response_json(dhcp4_config_with_shared_networks)) + source.fetch_and_set_dhcp_config() + assert source.kea_dhcp_config == KeaDhcpConfig.from_json(dhcp4_config_with_shared_networks) + + +# @pytest.fixture +# @enqueue_post_response +# def dhcp4_config_response_result_is_1(): +# return f''' +# {{ +# "result": 1, +# "arguments": {{ +# {DHCP4_CONFIG} +# }} +# }} +# ''' + +# def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): +# with pytest.raises(Exception): # TODO: Change +# get_dhcp_server("example-org", dhcp_version=4) + + +@pytest.fixture +def testlog(caplog): + caplog.clear() + caplog.set_level(logging.DEBUG) + return LogChecker(caplog) class LogChecker: @@ -16,12 +263,12 @@ def __init__(self, caplog): def clear(self): self.caplog.clear() - def has_entries(self, level, exception=None, regex=None, n=None): + def has_entries(self, level, exception=None, regexes=None, n=None): """ Check if there is any log entries of logging level `level`, optionally - made for an exception `exception`, optionally with message fully - matching regex `regex`, and optionally requiring that there is exactly - `n` such records logged. + made for an exception `exception`, optionally with all regexes in + `regexes` fully matching some substring of the log message, and + optionally requiring that there is exactly `n` such records logged. """ def causes(e: BaseException): @@ -32,23 +279,28 @@ def causes(e: BaseException): entries = [ entry for entry in self.caplog.records - if entry.levelno == level + if entry.levelno >= level and ( exception is None - and entry.exc_info is None or entry.exc_info is not None and exception in causes(entry.exc_info[1]) ) - and (regex is None or re.fullmatch(regex, entry.message.lower(), re.DOTALL)) + and (regexes is None or all(re.search(regex, entry.message.lower(), re.DOTALL) for regex in regexes)) ] return n is None and len(entries) > 0 or len(entries) == n -@pytest.fixture -def testlog(caplog): - caplog.clear() - caplog.set_level(logging.DEBUG) - return LogChecker(caplog) +def response_json(string): + return f''' + [ + {{ + "result": 0, + "arguments": {string} + }} + ] + ''' + + @pytest.fixture @@ -186,11 +438,11 @@ def dhcp4_config(): }] } } - ''' +''' @pytest.fixture -def dhcp4_config_w_shared_networks(): +def dhcp4_config_with_shared_networks(): return '''{ "Dhcp4": { "shared-networks": [ @@ -244,9 +496,8 @@ def enqueue_post_response(monkeypatch): This is how we mock what would otherwise be post requests to a server. """ - command_responses = ( - {} - ) # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K + # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K + command_responses = {} unknown_command_response = """[ {{ "result": 2, @@ -340,14 +591,6 @@ def invalid_json_response(): {"text": "b", "arguments": {"arg1": "val1"}, "service": "d" ]''' - -@pytest.fixture -def large_response(): - return ''' - - ''' - - def send_dummy_query(command="command"): return send_query( query=KeaQuery(command, [], {}), @@ -355,226 +598,7 @@ def send_dummy_query(command="command"): port=80, ) - -################################################################################ -# Testing the list[KeaResponse] returned by send_query() # -################################################################################ - - -def test_success_responses_does_succeed(success_response, enqueue_post_response): - enqueue_post_response("command", success_response) - responses = send_dummy_query("command") - assert len(responses) == 4 - for response in responses: - assert response.success - assert isinstance(response.text, str) - assert isinstance(response.arguments, dict) - assert isinstance(response.service, str) - - -def test_error_responses_does_not_succeed(error_response, enqueue_post_response): - enqueue_post_response("command", error_response) - responses = send_dummy_query("command") - assert len(responses) == 5 - for response in responses: - assert not response.success - assert isinstance(response.text, str) - assert isinstance(response.arguments, dict) - assert isinstance(response.service, str) - - -################################################################################ -# Testing correct error handling if Kea server returns invalid JSON # -################################################################################ - - -def test_invalid_json_response(testlog, invalid_json_response, enqueue_post_response): - enqueue_post_response("command", invalid_json_response) - testlog.clear() - with pytest.raises(KeaError): - responses = send_dummy_query("command") - assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") - - enqueue_post_response("config-get", lambda **_: invalid_json_response) - enqueue_post_response("statistic-get", lambda **_: invalid_json_response) - testlog.clear() - source = KeaDhcpMetricSource(address="192.0.2.1", port=80) - with pytest.raises(KeaError): - source.fetch_and_set_dhcp_config() - assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") - - # fetch_dhcp_config_hash should not raise when the server does not support - # config-hash-get command - testlog.clear() - h = source.fetch_dhcp_config_hash() - assert h == None - assert testlog.has_entries( - logging.DEBUG, regex=".*no.*support.*hash.*|.*hash.*no.*support.*" - ) - - # fetch_dhcp_config_hash should raise when the server returns invalid - # json - enqueue_post_response("config-hash-get", lambda **_: invalid_json_response) - testlog.clear() - with pytest.raises(KeaError): - source.fetch_and_set_dhcp_config() - assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") - - # FUNCTIONS USED EXTERNALLY SHOULD CATCH EXCEPTIONS AND LOG WARNINGS - # fetch_metrics is a method also used external to the module, and thus - # instead of raising it should log when the server returns invalid json - testlog.clear() - source.fetch_metrics() - assert testlog.has_entries(logging.WARNING, JSONDecodeError, n=1) - - -################################################################################ -# Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from json # -################################################################################ - - -def test_correct_subnet_from_dhcp4_config_json(dhcp4_config): - j = json.loads(dhcp4_config) - subnet = KeaDhcpSubnet.from_json(j["Dhcp4"]["subnet4"][0]) - assert subnet.id == 1 - assert subnet.prefix == IP("192.0.0.0/8") - assert len(subnet.pools) == 2 - assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) - assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) - - -def test_correct_config_from_dhcp4_config_json(dhcp4_config): - j = json.loads(dhcp4_config) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 1 - subnet = config.subnets[0] - assert subnet.id == 1 - assert subnet.prefix == IP("192.0.0.0/8") - assert len(subnet.pools) == 2 - assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) - assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - -def test_correct_config_from_dhcp6_config_json(dhcp6_config): - j = json.loads(dhcp6_config) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 2 - subnet1 = config.subnets[0] - assert subnet1.id == 1 - assert subnet1.prefix == IP("2001:db8:1:1::/64") - assert len(subnet1.pools) == 1 - assert subnet1.pools[0] == (IP("2001:db8:1:1::1"), IP("2001:db8:1:1::ffff")) - assert config.dhcp_version == 6 - assert config.config_hash is None - - subnet2 = config.subnets[1] - assert subnet2.id == 2 - assert subnet2.prefix == IP("2001:db8:1:2::/64") - assert len(subnet2.pools) == 2 - assert subnet2.pools[0] == (IP("2001:db8:1:2::1"), IP("2001:db8:1:2::ffff")) - assert subnet2.pools[1] == (IP("2001:db8:1:2::1:0"), IP("2001:db8:1:2::1:ffff")) - assert config.dhcp_version == 6 - assert config.config_hash is None - - -def test_correct_config_from_dhcp4_config_w_shared_networks_json( - dhcp4_config_w_shared_networks, -): - j = json.loads(dhcp4_config_w_shared_networks) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 4 - subnets = {subnet.id: subnet for subnet in config.subnets} - - subnet1 = subnets[1] - assert subnet1.id == 1 - assert subnet1.prefix == IP("192.0.1.0/24") - assert len(subnet1.pools) == 1 - assert subnet1.pools[0] == (IP("192.0.1.1"), IP("192.0.1.200")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - subnet2 = subnets[2] - assert subnet2.id == 2 - assert subnet2.prefix == IP("192.0.2.0/24") - assert len(subnet2.pools) == 1 - assert subnet2.pools[0] == (IP("192.0.2.100"), IP("192.0.2.199")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - subnet3 = subnets[3] - assert subnet3.id == 3 - assert subnet3.prefix == IP("192.0.3.0/24") - assert len(subnet3.pools) == 1 - assert subnet3.pools[0] == (IP("192.0.3.100"), IP("192.0.3.199")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - subnet4 = subnets[4] - assert subnet4.id == 4 - assert subnet4.prefix == IP("10.0.0.0/8") - assert len(subnet4.pools) == 1 - assert subnet4.pools[0] == (IP("10.0.0.1"), IP("10.0.0.99")) - assert config.dhcp_version == 4 - assert config.config_hash is None - -def response_json(dhcp4_config): - return f''' - [ - {{ - "result": 0, - "arguments": {dhcp4_config} - }} - ] - ''' - - -################################################################################ -# Now we assume KeaDhcpSubnet and KeaDhcpConfig instantiation from json is # -# correct. # -# Testing KeaDhcpSubnet and KeaDhcpConfig instantiation from server responses # -################################################################################ - - -def test_fetch_and_set_dhcp_config(dhcp4_config, enqueue_post_response): - enqueue_post_response("config-get", response_json(dhcp4_config)) - source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) - assert source.kea_dhcp_config is None - config = source.fetch_and_set_dhcp_config() - actual_config = KeaDhcpConfig.from_json( - json.loads(response_json(dhcp4_config))[0]["arguments"] - ) - assert config == actual_config - assert source.kea_dhcp_config == actual_config - - -def test_fetch_and_set_dhcp_config_w_shared_networks( - dhcp4_config_w_shared_networks, enqueue_post_response -): - enqueue_post_response("config-get", response_json(dhcp4_config_w_shared_networks)) - source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) - assert source.kea_dhcp_config is None - config = source.fetch_and_set_dhcp_config() - actual_config = KeaDhcpConfig.from_json( - json.loads(response_json(dhcp4_config_w_shared_networks))[0]["arguments"] - ) - assert config == actual_config - assert source.kea_dhcp_config == actual_config - - -# @pytest.fixture -# @enqueue_post_response -# def dhcp4_config_response_result_is_1(): -# return f''' -# {{ -# "result": 1, -# "arguments": {{ -# {DHCP4_CONFIG} -# }} -# }} -# ''' - -# def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): -# with pytest.raises(Exception): # TODO: Change -# get_dhcp_server("example-org", dhcp_version=4) +def raiser(exception: type[Exception]): + def do_raise(*args, **kwargs): + raise exception + return do_raise From bf1e7dfd69a47e10f27f5b9a09a2605d27c24bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Thu, 4 Jul 2024 15:11:02 +0200 Subject: [PATCH 35/77] Simplify kea dhcp module and remove unnecessary cruft --- python/nav/dhcp/kea_metrics.py | 571 +++++++-------------------------- 1 file changed, 114 insertions(+), 457 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 7911d3acd7..147978f926 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -1,490 +1,147 @@ -""" -Functions for querying the Kea Control Agent for statistics from Kea DHCP -servers. - - RESTful-queries IPC -nav <---------------> Kea Control Agent <=====> Kea DHCP4 server / Kea DHCP6 server - send_query() - -No additional hook libraries are assumed to be included with the Kea Control -Agent that is queried, meaning this module will be able to gather DHCP -statistics from any Kea Control Agent. - -* Stork (https://gitlab.isc.org/isc-projects/stork) is used as a guiding -implementation for interacting with the Kea Control Agent. -* See also the Kea Control Agent documentation. This script assumes Kea versions ->= 2.2.0 are used. (https://kea.readthedocs.io/en/kea-2.2.0/arm/agent.html). -""" -from __future__ import annotations -import calendar -import json -import logging -import requests -import time -from dataclasses import dataclass, asdict -from enum import IntEnum from IPy import IP -from nav.dhcp.generic_metrics import DhcpMetricSource, DhcpMetric, DhcpMetricKey +from typing import Iterator, Optional +from itertools import chain +from nav.dhcp.generic_metrics import DhcpMetricSource from nav.errors import GeneralException -from requests.exceptions import JSONDecodeError, HTTPError, Timeout -from typing import Optional - -_logger = logging.getLogger(__name__) +import logging +logger = logging.getLogger(__name__) class KeaDhcpMetricSource(DhcpMetricSource): - """ - Using `send_query()`, this class: - * Maintains an up-to-date `KeaDhcpConfig` representation of the - configuration of the Kea DHCP server with ip version - `self.dhcp_version` reachable via the Kea Control Agent listening - to port `self.rest_port` on IP addresses `self.rest_address` - * Queries the Kea Control Agent for statistics about each subnet - found in the `KeaDhcpConfig` representation and creates an - iterable of `DhcpMetric` that its superclass uses to fill a - graphite server with metrics. - """ - - rest_address: str # IP address of the Kea Control Agent server - rest_port: int # Port of the Kea Control Agent server - rest_https: bool # If true, communicate with Kea Control Agent using https. If false, use http. - - dhcp_version: int # The IP version of the Kea DHCP server. The Kea Control Agent uses this to tell if we want information from its IPv6 or IPv4 Kea DHCP server - kea_dhcp_config: KeaDhcpConfig # The configuration, i.e. most static pieces of information, of the Kea DHCP server. + dhcp_config: dict + dhcp_confighash: Optional[str] + dhcp_version: int + rest_url: str def __init__( - self, - address: str, - port: int, - *args, - https: bool = True, - dhcp_version: int = 4, - **kwargs, + self, + address: str, + port: int, + *args, + https: bool = True, + dhcp_version: int = 4, + timeout = 10, + **kwargs, ): super(*args, **kwargs) - self.rest_address = address - self.rest_port = port - self.rest_https = https - self.dchp_version = dhcp_version - self.kea_dhcp_config = None + scheme = "https" if https else "http" + self.rest_url = f"{scheme}://{address}:{port}/" + self.dhcp_version = dhcp_version + self.dchp_confighash = None + + def fetch_metrics(self) -> Iterator[DhcpMetric]: + config = self.fetch_config() - def fetch_metrics(self) -> list[DhcpMetric]: - """ - Implementation of the superclass method for fetching - standardised dhcp metrics. This method is used by the - superclass to feed data into a graphite server. - """ metrics = [] - try: - s = requests.Session() - self.fetch_and_set_dhcp_config(s) - for subnet in self.kea_dhcp_config.subnets: + with requests.Session as s: + for subnetid, prefix in subnets_of_config(config): for kea_key, nav_key in ( - ("total-addresses", DhcpMetricKey.MAX), - ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH), + ("total-addresses", DhcpMetricKey.MAX), + ("assigned-addresses", DhcpMetricKey.CUR), + ("declined-addresses", DhcpMetricKey.TOUCH), ): - kea_statistic_name = f"subnet[{subnet.id}].{kea_key}" - query = KeaQuery( - command="statistic-get", - service=[f"dhcp{self.dchp_version}"], - arguments={ - "name": kea_statistic_name, - }, - ) - response = unwrap( - send_query( - query, - self.rest_address, - self.rest_port, - self.rest_https, - session=s, - ) - ) - - datapoints = response["arguments"].get(kea_statistic_name, []) - for value, timestamp in datapoints: - # Assumes for now that UTC timestamps are returned by Kea Control Agent; I'll need to read the documentation closer! - epochseconds = calendar.timegm( - time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + kea_statisticname = f"subnet[{subnetid}].{kea_key}" + response = self.send_query(session, "statistic-get", name=kea_statisticname) + timeseries = response.get("arguments", {}).get(kea_statisticname, []) + if len(timeseries) == 0: + logger.error( + "fetch_metrics: Could not fetch metric '%r' for subnet " + "'%s' from Kea: '%s' from Kea is an empty list.", + nav_key, prefix, kea_statisticname, ) + continue + for value, timestamp in timeseries: metrics.append( - DhcpMetric(epochseconds, subnet.prefix, nav_key, value) + DhcpMetric(parsetime(timestamp), prefix, nav_key, value) ) - used_config = self.kea_dhcp_config - self.fetch_and_set_dhcp_config(s) - if sorted(used_config.subnets) != sorted(self.kea_dhcp_config.subnets): - _logger.warning( - "Subnet configuration was modified during metric fetching, " - "this may cause metric data being associated with wrong " - "subnet." - ) - except KeaError as err: - _logger.warning( - "Error while fetching metrics from Kea Control Agent. Some " - "metrics may be missing.", - exc_info=err, + if sorted(subnets_of_config(config)) != sorted(subnets_of_config(self.fetch_config())): + logger.error( + "Subnet configuration was modified during metric fetching, " + "this may cause metric data being associated with wrong " + "subnet." ) - finally: - s.close() - + return metrics - def fetch_and_set_dhcp_config(self, session=None) -> KeaDhcpConfig: - """ - Fetch the current config used by the Kea DHCP server that - manages addresses of IP version `self.dhcp_version` from the Kea - Control Agent listening to `self.rest_port` on - `self.rest_address`. - """ - # Check if self.kea_dhcp_config is up to date - if not ( - self.kea_dhcp_config is None - or self.kea_dhcp_config.config_hash is None - or self.fetch_dhcp_config_hash(session=session) - != self.kea_dhcp_config.config_hash - ): - return self.kea_dhcp_config - - # self.kea_dhcp_config is not up to date, fetch new - query = KeaQuery( - command="config-get", - service=[f"dhcp{self.dchp_version}"], - arguments={}, + def self.send_query(self, session, command, **kwargs) -> dict: + postdata = json.dumps({ + "command": command, + "arguments": **kwargs, + "service": [f"dhcp{dhcp_version}"] + }) + logger.info("send_query: Post request to Kea with data %s", postdata) + r = session.post( + self.rest_url, + data=postdata, + headers=self.rest_headers, + timeout=timeout, ) - response = unwrap( - send_query( - query, - self.rest_address, - self.rest_port, - self.rest_https, - session=session, + rjson = r.json() + if not isinstance(rjson, list): + # See https://kea.readthedocs.io/en/kea-2.6.0/arm/ctrl-channel.html#control-agent-command-response-format + raise KeaError( + "send_query: Kea have likely rejected a query (responsed with: {rjson!r})" ) - ) - - self.kea_dhcp_config = KeaDhcpConfig.from_json(response.arguments) - return self.kea_dhcp_config - - def fetch_dhcp_config_hash(self, session=None): - """ - For Kea versions >= 2.4.0, fetch and return a hash of the - current configuration used by the Kea DHCP server. For Kea - versions < 2.4.0, return None. - """ - query = KeaQuery( - command="config-hash-get", - service=[f"dhcp{self.dchp_version}"], - arguments={}, - ) - response = unwrap( - send_query( - query, - self.rest_address, - self.rest_port, - self.rest_https, - session=session, - ), - require_success=False, - ) - - if response.success: - return response.arguments.get("hash", None) - elif response.result == KeaStatus.UNSUPPORTED: - _logger.debug("Kea server does not support command 'config-hash-get'") - return None + assert len(rjson) == 1 + response = rjson.pop() + if response.get("result", KeaStatus.ERROR) == KeaStatus.SUCCESS + return response else: - _logger.debug( - "Unexpected response (%s) after querying the hash of config file from Kea " - "DHCP%d server", - repr(response), - self.dhcp_version, - ) - return None - - -@dataclass -class KeaDhcpConfig: - """ - Class representing information found in the configuration of a Kea DHCP - server. Most importantly, this class contains: - * A list of the subnets managed by the DHCP server - * The IP version of the DCHP server - """ - - # Used to check if there's a new config on the Kea DHCP server - config_hash: Optional[str] - dhcp_version: int - subnets: list[KeaDhcpSubnet] - - @classmethod - def from_json( - cls, - config_json: dict, - config_hash: Optional[str] = None, + logger.error("send_query: Kea did not succeed fulfilling a query (responded with: {rjson!r}) ") + return {} + +def parsetime(timestamp: str) -> int: + return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) + +def subnets_of_config(config: dict) -> Iterator[tuple[int, IP]]: + if "Dhcp4" in config: + dhcpversion = 4 + config = config["Dhcp4"] + subnetkey = "subnet4" + elif "Dhcp6" in config: + dhcpversion = 6 + config = config["Dhcp6"] + subnetkey = "subnet6" + else: + logger.warning("subnets: expected a Kea Dhcp4 or Kea Dhcp6 config, got: ", config) + return + + for subnet in chain.from_iterable( + [config.get(subnetkey, [])] + + [network.get(subnetkey, []) for network in config.get("shared-networks", [])] ): - """ - Initialize and return a KeaDhcpConfig instance based on json - - :param json: a dictionary that is structured the same way as a - Kea DHCP configuration. - Example: - { - "Dhcp4": { - "shared-networks": [ - { - "name": "test-network", - "subnet4": [ - { - "subnet": "10.0.0.0/8", - "pools": [ { "pool": "10.0.0.1 - 10.0.0.99" } ], - }, - { - "subnet": "192.0.3.0/24", - "pools": [ { "pool": "192.0.3.100 - 192.0.3.199" } ] - } - ], - } - ], # end of shared-networks - "subnet4": [{ - "id": 1, - "subnet": "192.0.2.0/24", - "pools": [ - { - "pool": "192.0.2.1 - 192.0.2.200", - }, - ], - }] # end of subnet4 - } - } - - :param hash: hash of the Kea DHCP config file as returned by a - `config-hash-get` query on the kea-ctrl-agent REST server. - """ - if len(config_json) != 1: - _logger.debug( - "KeaDhcpConfig.from_json: expected outermost object to have one key, got: %r", - config_json, - ) - raise KeaError("Invalid DHCP config JSON") - - service, config_json = config_json.popitem() - if service == "Dhcp4": - dhcp_version = 4 - elif service == "Dhcp6": - dhcp_version = 6 - else: - _logger.debug( - "KeaDhcpConfig.from_json: config JSON from unknown Kea service: %s", - service, - ) - raise KeaError(f"Unsupported Kea service '{service}'") - - subnets = [] - for obj in config_json.get(f"subnet{dhcp_version}", []): - subnet = KeaDhcpSubnet.from_json(obj) - subnets.append(subnet) - for obj in config_json.get("shared-networks", []): - for subobj in obj.get(f"subnet{dhcp_version}", []): - subnet = KeaDhcpSubnet.from_json(subobj) - subnets.append(subnet) - - return cls( - config_hash=config_hash, - dhcp_version=dhcp_version, - subnets=subnets, - ) - - -@dataclass -class KeaDhcpSubnet: - """Class representing information about a subnet managed by a Kea DHCP server.""" - - id: int # either specified in the server config or assigned automatically by the dhcp server - prefix: IP # e.g. 192.0.2.1/24 - pools: list[tuple[IP]] # e.g. [(192.0.2.10, 192.0.2.20), (192.0.2.64, 192.0.2.128)] - - @classmethod - def from_json(cls, subnet_json: dict): - """ - Initialize and return a Subnet instance based on json - - :param json: python dictionary that is structured the same way as the - json object representing a subnet in the Kea DHCP config file. - Example: - { - "id": 0 - "subnet": "192.0.2.0/24", - "pools": [ - { - "pool": "192.0.2.1 - 192.0.2.100" - }, - { - "pool": "192.0.2.128/26" - } - ] - } - """ - if "id" not in subnet_json: - _logger.debug( - "KeaDhcpSubnet.from_json: subnet JSON missing key 'id': %r", subnet_json - ) - raise KeaError("Expected subnetjson['id'] to exist") - id = subnet_json["id"] - - if "subnet" not in subnet_json: - _logger.debug( - "KeaDhcpSubnet.from_json: subnet JSON missing key 'subnet': %r", - subnet_json, - ) - raise KeaError("Expected subnetjson['subnet'] to exist") - prefix = IP(subnet_json["subnet"]) - - pools = [] - for obj in subnet_json.get("pools", []): - pool = obj["pool"] - if "-" in pool: # TODO: Error checking? - # pool == "x.x.x.x - y.y.y.y" - start, end = (IP(ip) for ip in pool.split("-")) - else: - # pool == "x.x.x.x/nn" - pool = IP(pool) - start, end = pool[0], pool[-1] - pools.append((start, end)) - - return cls( - id=id, - prefix=prefix, - pools=pools, - ) - - -def send_query( - query: KeaQuery, - address: str, - port: int = 443, - https: bool = True, - session: requests.Session = None, - timeout: int = 10, -) -> list[KeaResponse]: - """ - Send `query` to a Kea Control Agent listening to `port` on IP - address `address`, using either http or https - - :param https: If True, use https. Otherwise, use http. - - :param session: Optional requests.Session to be used when sending - the query. Assumed to not be closed. session is not closed after - the end of this call, so that session can be used for persistent - http connections among different send_query calls. - """ - scheme = "https" if https else "http" - location = f"{scheme}://{address}:{port}/" - _logger.debug("send_query: sending request to %s with query %r", location, query) - try: - if session is None: - r = requests.post( - location, - data=json.dumps(asdict(query)), - headers={"Content-Type": "application/json"}, - timeout=timeout, - ) - else: - r = session.post( - location, - data=json.dumps(asdict(query)), - headers={"Content-Type": "application/json"}, - timeout=timeout, - ) - except RequestException as err: - _logger.debug( - "send_query: Requests failed to complete request to %s with query %r", - location, - query, - ) - raise KeaError() from err - - try: - response_json = r.json() - except JSONDecodeError as err: - _logger.debug( - "send_query: invalid json from %s, got: %s", - address, - r.text, - ) - raise KeaError() from err - - if isinstance(response_json, dict): - _logger.debug( - "send_query: expected a json list of objects from %s, got: %r", - address, - response_json, - ) - raise KeaError(f"bad response from {address}: {response_json!r}") - - responses = [] - for obj in response_json: - response = KeaResponse( - obj.get("result", KeaStatus.ERROR), - obj.get("text", ""), - obj.get("arguments", {}), - obj.get("service", ""), - ) - responses.append(response) - return responses - - -def unwrap(responses: list[KeaQuery], require_success=True) -> KeaQuery: - """ - Helper function implementing the sequence of operations often done - on the list of responses returned by `send_query()` - """ - if len(responses) != 1: - _logger.debug("unwrap: received %d responses, expected 1 (%r)", responses) - raise KeaError("Received invalid amount of responses") - - response = responses[0] - if require_success and not response.success: - _logger.debug("unwrap: received an unsuccessful response (%r)", response) - raise KeaError("Did not receive a successful response") - return response - - -@dataclass -class KeaResponse: - """ - Class representing the response to a REST query sent to a Kea - Control Agent. - """ - - result: int - text: str - arguments: dict - service: str - - @property - def success(self) -> bool: - return self.result == KeaStatus.SUCCESS - - -@dataclass -class KeaQuery: - """Class representing a REST query to be sent to a Kea Control Agent.""" - - command: str - arguments: dict - - # The server(s) at which the command is targeted. Usually ["dhcp4", "dhcp6"] or ["dhcp4"] or ["dhcp6"]. - service: list[str] - + id = subnet.get("id", None) + prefix = subnet.get("subnet", None) + if id is None or prefix is None: + logger.warning("subnets: id or prefix missing from a subnet's configuration: %r", subnet) + continue + yield id, IP(prefix) + +###> +d = {"Dhcp4": {}} +d = {"Dhcp4": {"subnet4": []}} +d = {"Dhcp4": {"subnet4": [{"subnet": "192.0.2.0/24", "id": 1}]}} +d = {"Dhcp4": {"subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}]}} +d = {"Dhcp4": {"subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}, {"subnet": "10.2.0.0/17", "id": 3}]}} +d = {"Dhcp4": { + "subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}, {"subnet": "10.2.0.0/17", "id": 3}], + "shared-networks": []}} +d = {"Dhcp4": { + "shared-networks": [{"subnet4": [{"subnet": "192.0.3.0/25", "id": 4}, {"subnet": "192.0.4.0/29", "id": 5}]}]}} +d = {"Dhcp4": { + "subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}, {"subnet": "10.2.0.0/17", "id": 3}], + "shared-networks": [{"subnet4": [{"subnet": "192.0.3.0/25", "id": 4}, {"subnet": "192.0.4.0/29", "id": 5}]}]}} + +for subnet in subnets_of_config(d): + print(repr(subnet)) +###< class KeaError(GeneralException): """Error related to interaction with a Kea Control Agent""" - class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" - # Successful operation. SUCCESS = 0 # General failure. From 6e3a2bfa91a108d7beabef768614be470f2203af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 5 Jul 2024 11:11:14 +0200 Subject: [PATCH 36/77] Update error checking in `send_query()` An exception is raised iff there was an HTTP related error while sending a command or a response does not look like it is coming from a Kea Control Agent. All raised exceptions are of type KeaError. Proper error responses as documented in the API are logged but results in an empty dictionary being returned (not an exception being thrown). --- python/nav/dhcp/kea_metrics.py | 69 ++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 147978f926..fdcaee8d19 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -64,31 +64,70 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: return metrics - def self.send_query(self, session, command, **kwargs) -> dict: + + def fetch_config(self): + raise NotImplementedError + + + def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: + """ + Send `command` to the Kea Control Agent. An exception is raised iff + there was an HTTP related error while sending `command` or a response + does not look like it is coming from a Kea Control Agent. All raised + exceptions are of type `KeaError`. Proper error responses as documented + in the API are logged but results in an empty dictionary being returned. + """ postdata = json.dumps({ "command": command, "arguments": **kwargs, - "service": [f"dhcp{dhcp_version}"] + "service": [f"dhcp{self.dhcp_version}"] }) - logger.info("send_query: Post request to Kea with data %s", postdata) - r = session.post( - self.rest_url, - data=postdata, - headers=self.rest_headers, - timeout=timeout, + logger.info( + "send_query: Post request to Kea Control Agent at %s with data %s", + self.rest_uri, + postdata, ) - rjson = r.json() - if not isinstance(rjson, list): + try: + responses = session.post( + self.rest_uri, + data=postdata, + headers=self.rest_headers, + timeout=self.timeout, + ) + responses = responses.json() + except RequestException as err: + raise KeaError( + f"HTTP related error when requesting Kea Control Agent at {self.rest_uri}", + ) from err + except JSONDecodeError as err: + raise KeaError( + f"Uri {self.rest_uri} most likely not pointing at a Kea " + f"Control Agent (expected json, responded with: {responses!r})", + ) from err + if not isinstance(responses, list): # See https://kea.readthedocs.io/en/kea-2.6.0/arm/ctrl-channel.html#control-agent-command-response-format raise KeaError( - "send_query: Kea have likely rejected a query (responsed with: {rjson!r})" + f"Kea Control Agent at {self.rest_uri} have likely rejected " + f"a query (responded with: {rjson!r})" + ) + if not (len(responses) == 1 and "result" in responses[0]): + # "We've only sent the command to *one* service. Thus responses should contain *one* response." + raise KeaError( + f"Uri {self.rest_uri} most likely not pointing at a Kea " + "Control Agent (expected json list with one object having " + f"key 'result', responded with: {responses!r})", ) - assert len(rjson) == 1 - response = rjson.pop() - if response.get("result", KeaStatus.ERROR) == KeaStatus.SUCCESS + response = responses[0] + if response["result"] == KeaStatus.SUCCESS return response else: - logger.error("send_query: Kea did not succeed fulfilling a query (responded with: {rjson!r}) ") + logger.error( + "send_query: Kea at %s did not succeed fulfilling query %s " + "(responded with: %r) ", + self.rest_uri, + postdata, + responses + ) return {} def parsetime(timestamp: str) -> int: From 51b7001094ed3c1afc19205f0b3a456a5462720b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 5 Jul 2024 11:16:18 +0200 Subject: [PATCH 37/77] Add docstrings --- python/nav/dhcp/kea_metrics.py | 42 ++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index fdcaee8d19..6627986fa7 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -8,10 +8,15 @@ logger = logging.getLogger(__name__) class KeaDhcpMetricSource(DhcpMetricSource): + """ + Sends http requests to Kea Control Agent on `self.rest_uri` to fetch metrics + from all subnets managed by the Kea DHCP server (serving ip version + `dhcp_version` addresses) that the Kea Control Agent controls. + """ dhcp_config: dict dhcp_confighash: Optional[str] dhcp_version: int - rest_url: str + rest_uri: str def __init__( self, @@ -23,25 +28,35 @@ def __init__( timeout = 10, **kwargs, ): + """ + Instantiate a KeaDhcpMetricSource that fetches information via the Kea + Control Agent listening to `port` on `address`. + """ super(*args, **kwargs) scheme = "https" if https else "http" - self.rest_url = f"{scheme}://{address}:{port}/" + self.rest_uri = f"{scheme}://{address}:{port}/" self.dhcp_version = dhcp_version self.dchp_confighash = None def fetch_metrics(self) -> Iterator[DhcpMetric]: + """ + Fetch total addresses, assigned addresses, and declined addresses of all + subnets the Kea DHCP server serving ip version `dhcp_version` maintains. + """ config = self.fetch_config() - + subnets = subnets_of_config(config) + metric_keys = ( + ("total-addresses", DhcpMetricKey.MAX), + ("assigned-addresses", DhcpMetricKey.CUR), + ("declined-addresses", DhcpMetricKey.TOUCH), + ) metrics = [] + with requests.Session as s: - for subnetid, prefix in subnets_of_config(config): - for kea_key, nav_key in ( - ("total-addresses", DhcpMetricKey.MAX), - ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH), - ): + for subnetid, prefix in subnets: + for kea_key, nav_key in metric_keys: kea_statisticname = f"subnet[{subnetid}].{kea_key}" - response = self.send_query(session, "statistic-get", name=kea_statisticname) + response = self.send_query(s, "statistic-get", name=kea_statisticname) timeseries = response.get("arguments", {}).get(kea_statisticname, []) if len(timeseries) == 0: logger.error( @@ -49,19 +64,18 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: "'%s' from Kea: '%s' from Kea is an empty list.", nav_key, prefix, kea_statisticname, ) - continue for value, timestamp in timeseries: metrics.append( DhcpMetric(parsetime(timestamp), prefix, nav_key, value) ) - if sorted(subnets_of_config(config)) != sorted(subnets_of_config(self.fetch_config())): + if sorted(subnets) != sorted(subnets_of_config(self.fetch_config())): logger.error( "Subnet configuration was modified during metric fetching, " "this may cause metric data being associated with wrong " - "subnet." + "subnet in some rare cases." ) - + return metrics From 27161a7b047d39edf64f08d1cc12051776b46aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 5 Jul 2024 16:58:08 +0200 Subject: [PATCH 38/77] Implement `fetch_config()` and `fetch_config_hash()` --- python/nav/dhcp/kea_metrics.py | 77 +++++++++++++++------------------- 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 6627986fa7..a750f63502 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -5,7 +5,7 @@ from nav.errors import GeneralException import logging -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class KeaDhcpMetricSource(DhcpMetricSource): """ @@ -14,7 +14,6 @@ class KeaDhcpMetricSource(DhcpMetricSource): `dhcp_version` addresses) that the Kea Control Agent controls. """ dhcp_config: dict - dhcp_confighash: Optional[str] dhcp_version: int rest_uri: str @@ -36,15 +35,14 @@ def __init__( scheme = "https" if https else "http" self.rest_uri = f"{scheme}://{address}:{port}/" self.dhcp_version = dhcp_version - self.dchp_confighash = None + self.dchp_config = None def fetch_metrics(self) -> Iterator[DhcpMetric]: """ Fetch total addresses, assigned addresses, and declined addresses of all subnets the Kea DHCP server serving ip version `dhcp_version` maintains. """ - config = self.fetch_config() - subnets = subnets_of_config(config) + metric_keys = ( ("total-addresses", DhcpMetricKey.MAX), ("assigned-addresses", DhcpMetricKey.CUR), @@ -53,13 +51,15 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: metrics = [] with requests.Session as s: + config = self.fetch_config(s) + subnets = subnets_of_config(config, self.dhcp_version) for subnetid, prefix in subnets: for kea_key, nav_key in metric_keys: kea_statisticname = f"subnet[{subnetid}].{kea_key}" response = self.send_query(s, "statistic-get", name=kea_statisticname) timeseries = response.get("arguments", {}).get(kea_statisticname, []) if len(timeseries) == 0: - logger.error( + _logger.error( "fetch_metrics: Could not fetch metric '%r' for subnet " "'%s' from Kea: '%s' from Kea is an empty list.", nav_key, prefix, kea_statisticname, @@ -70,7 +70,7 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: ) if sorted(subnets) != sorted(subnets_of_config(self.fetch_config())): - logger.error( + _logger.error( "Subnet configuration was modified during metric fetching, " "this may cause metric data being associated with wrong " "subnet in some rare cases." @@ -79,8 +79,28 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: return metrics - def fetch_config(self): - raise NotImplementedError + def fetch_config(self, session: requests.Session): + """ + Fetch the current config of the Kea DHCP server serving ip version + `dhcp_version` + """ + if ( + self.dhcp_config is None + or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None + or self.fetch_config_hash(session) != dchp_confighash + ): + self.dhcp_config = self.send_query(session, "config-get").get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) + if self.dhcp_config is None: + raise KeaError( + "Could not fetch configuration of Kea DHCP server from Kea Control " + f"Agent at {self.rest_uri}" + ) + + return self.dhcp_config + + + def fetch_config_hash(self, session: requests.Session): + return self.send_query(session, "config-hash-get").get("arguments", {}).get("hash", None) def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: @@ -96,7 +116,7 @@ def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: "arguments": **kwargs, "service": [f"dhcp{self.dhcp_version}"] }) - logger.info( + _logger.info( "send_query: Post request to Kea Control Agent at %s with data %s", self.rest_uri, postdata, @@ -135,7 +155,7 @@ def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: if response["result"] == KeaStatus.SUCCESS return response else: - logger.error( + _logger.error( "send_query: Kea at %s did not succeed fulfilling query %s " "(responded with: %r) ", self.rest_uri, @@ -146,20 +166,9 @@ def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: def parsetime(timestamp: str) -> int: return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) - -def subnets_of_config(config: dict) -> Iterator[tuple[int, IP]]: - if "Dhcp4" in config: - dhcpversion = 4 - config = config["Dhcp4"] - subnetkey = "subnet4" - elif "Dhcp6" in config: - dhcpversion = 6 - config = config["Dhcp6"] - subnetkey = "subnet6" - else: - logger.warning("subnets: expected a Kea Dhcp4 or Kea Dhcp6 config, got: ", config) - return +def subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP]]: + subnetkey = f"subnet{ip_version}" for subnet in chain.from_iterable( [config.get(subnetkey, [])] + [network.get(subnetkey, []) for network in config.get("shared-networks", [])] @@ -167,28 +176,10 @@ def subnets_of_config(config: dict) -> Iterator[tuple[int, IP]]: id = subnet.get("id", None) prefix = subnet.get("subnet", None) if id is None or prefix is None: - logger.warning("subnets: id or prefix missing from a subnet's configuration: %r", subnet) + _logger.warning("subnets: id or prefix missing from a subnet's configuration: %r", subnet) continue yield id, IP(prefix) -###> -d = {"Dhcp4": {}} -d = {"Dhcp4": {"subnet4": []}} -d = {"Dhcp4": {"subnet4": [{"subnet": "192.0.2.0/24", "id": 1}]}} -d = {"Dhcp4": {"subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}]}} -d = {"Dhcp4": {"subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}, {"subnet": "10.2.0.0/17", "id": 3}]}} -d = {"Dhcp4": { - "subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}, {"subnet": "10.2.0.0/17", "id": 3}], - "shared-networks": []}} -d = {"Dhcp4": { - "shared-networks": [{"subnet4": [{"subnet": "192.0.3.0/25", "id": 4}, {"subnet": "192.0.4.0/29", "id": 5}]}]}} -d = {"Dhcp4": { - "subnet4": [{"subnet": "192.0.2.0/24", "id": 1}, {"subnet": "10.1.0.0/16", "id": 2}, {"subnet": "10.2.0.0/17", "id": 3}], - "shared-networks": [{"subnet4": [{"subnet": "192.0.3.0/25", "id": 4}, {"subnet": "192.0.4.0/29", "id": 5}]}]}} - -for subnet in subnets_of_config(d): - print(repr(subnet)) -###< class KeaError(GeneralException): """Error related to interaction with a Kea Control Agent""" From 07dbb959116f76a37492925d7b30eeca658ffe4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 5 Jul 2024 17:13:32 +0200 Subject: [PATCH 39/77] Add docstrings --- python/nav/dhcp/kea_metrics.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index a750f63502..36b35efcf4 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -100,6 +100,10 @@ def fetch_config(self, session: requests.Session): def fetch_config_hash(self, session: requests.Session): + """ + Fetch the hash of the current config of the Kea DHCP server serving ip + version `dhcp_version` + """ return self.send_query(session, "config-hash-get").get("arguments", {}).get("hash", None) @@ -165,9 +169,14 @@ def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: return {} def parsetime(timestamp: str) -> int: + """Parse the timestamp string used in Kea's timeseries into unix time""" return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) def subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP]]: + """ + List the id and prefix of subnets listed in the Kea DHCP configuration + `config` + """ subnetkey = f"subnet{ip_version}" for subnet in chain.from_iterable( [config.get(subnetkey, [])] From c590723582e386f6c78c1f103114fa6896e3705c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 09:20:29 +0200 Subject: [PATCH 40/77] Remove class variables --- python/nav/dhcp/kea_metrics.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 36b35efcf4..c036a8b3f6 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -13,10 +13,6 @@ class KeaDhcpMetricSource(DhcpMetricSource): from all subnets managed by the Kea DHCP server (serving ip version `dhcp_version` addresses) that the Kea Control Agent controls. """ - dhcp_config: dict - dhcp_version: int - rest_uri: str - def __init__( self, address: str, From 6c92967f29710ae10ef3c89bf82dbc60d1c62d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 09:41:20 +0200 Subject: [PATCH 41/77] Start private (helper) functions' names with underscore --- python/nav/dhcp/kea_metrics.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index c036a8b3f6..6cbe10f330 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -31,7 +31,7 @@ def __init__( scheme = "https" if https else "http" self.rest_uri = f"{scheme}://{address}:{port}/" self.dhcp_version = dhcp_version - self.dchp_config = None + self.dchp_config: Optional[dict] = None def fetch_metrics(self) -> Iterator[DhcpMetric]: """ @@ -47,12 +47,12 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: metrics = [] with requests.Session as s: - config = self.fetch_config(s) - subnets = subnets_of_config(config, self.dhcp_version) + config = self._fetch_config(s) + subnets = _subnets_of_config(config, self.dhcp_version) for subnetid, prefix in subnets: for kea_key, nav_key in metric_keys: kea_statisticname = f"subnet[{subnetid}].{kea_key}" - response = self.send_query(s, "statistic-get", name=kea_statisticname) + response = self._send_query(s, "statistic-get", name=kea_statisticname) timeseries = response.get("arguments", {}).get(kea_statisticname, []) if len(timeseries) == 0: _logger.error( @@ -62,10 +62,10 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: ) for value, timestamp in timeseries: metrics.append( - DhcpMetric(parsetime(timestamp), prefix, nav_key, value) + DhcpMetric(_parsetime(timestamp), prefix, nav_key, value) ) - if sorted(subnets) != sorted(subnets_of_config(self.fetch_config())): + if sorted(subnets) != sorted(_subnets_of_config(self._fetch_config())): _logger.error( "Subnet configuration was modified during metric fetching, " "this may cause metric data being associated with wrong " @@ -75,7 +75,7 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: return metrics - def fetch_config(self, session: requests.Session): + def _fetch_config(self, session: requests.Session) -> dict: """ Fetch the current config of the Kea DHCP server serving ip version `dhcp_version` @@ -83,9 +83,9 @@ def fetch_config(self, session: requests.Session): if ( self.dhcp_config is None or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None - or self.fetch_config_hash(session) != dchp_confighash + or self._fetch_config_hash(session) != dchp_confighash ): - self.dhcp_config = self.send_query(session, "config-get").get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) + self.dhcp_config = self._send_query(session, "config-get").get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) if self.dhcp_config is None: raise KeaError( "Could not fetch configuration of Kea DHCP server from Kea Control " @@ -95,15 +95,15 @@ def fetch_config(self, session: requests.Session): return self.dhcp_config - def fetch_config_hash(self, session: requests.Session): + def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: """ Fetch the hash of the current config of the Kea DHCP server serving ip version `dhcp_version` """ - return self.send_query(session, "config-hash-get").get("arguments", {}).get("hash", None) + return self._send_query(session, "config-hash-get").get("arguments", {}).get("hash", None) - def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: + def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict: """ Send `command` to the Kea Control Agent. An exception is raised iff there was an HTTP related error while sending `command` or a response @@ -164,11 +164,11 @@ def send_query(self, session: requests.Session, command: str, **kwargs) -> dict: ) return {} -def parsetime(timestamp: str) -> int: +def _parsetime(timestamp: str) -> int: """Parse the timestamp string used in Kea's timeseries into unix time""" return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) -def subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP]]: +def _subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP]]: """ List the id and prefix of subnets listed in the Kea DHCP configuration `config` From 0b2d5c7ec112d1a0bf0d93e0e276d8f6f4b3a97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 11:33:24 +0200 Subject: [PATCH 42/77] Add imports --- python/nav/dhcp/kea_metrics.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 6cbe10f330..2c68e9e486 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -3,7 +3,14 @@ from itertools import chain from nav.dhcp.generic_metrics import DhcpMetricSource from nav.errors import GeneralException +from nav.dhcp.generic_metrics import DhcpMetric, DhcpMetricKey, DhcpMetricSource import logging +from requests import RequestException, JSONDecodeError +import requests +import calendar +import time +import json +from enum import IntEnum _logger = logging.getLogger(__name__) From 760e44d6575a3855ddf51fe8369b2ecc41e69055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 11:35:07 +0200 Subject: [PATCH 43/77] Fix naming and syntax errors --- python/nav/dhcp/kea_metrics.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 2c68e9e486..b6ecec7048 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -38,7 +38,7 @@ def __init__( scheme = "https" if https else "http" self.rest_uri = f"{scheme}://{address}:{port}/" self.dhcp_version = dhcp_version - self.dchp_config: Optional[dict] = None + self.dhcp_config: Optional[dict] = None def fetch_metrics(self) -> Iterator[DhcpMetric]: """ @@ -88,9 +88,9 @@ def _fetch_config(self, session: requests.Session) -> dict: `dhcp_version` """ if ( - self.dhcp_config is None - or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None - or self._fetch_config_hash(session) != dchp_confighash + self.dhcp_config is None + or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None + or self._fetch_config_hash(session) != dhcp_confighash ): self.dhcp_config = self._send_query(session, "config-get").get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) if self.dhcp_config is None: @@ -149,7 +149,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict # See https://kea.readthedocs.io/en/kea-2.6.0/arm/ctrl-channel.html#control-agent-command-response-format raise KeaError( f"Kea Control Agent at {self.rest_uri} have likely rejected " - f"a query (responded with: {rjson!r})" + f"a query (responded with: {responses!r})" ) if not (len(responses) == 1 and "result" in responses[0]): # "We've only sent the command to *one* service. Thus responses should contain *one* response." @@ -159,7 +159,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict f"key 'result', responded with: {responses!r})", ) response = responses[0] - if response["result"] == KeaStatus.SUCCESS + if response["result"] == KeaStatus.SUCCESS: return response else: _logger.error( From b05e6f8a73d77654db1a86d85cc9d0076b0cb2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 11:39:52 +0200 Subject: [PATCH 44/77] Fix linting errors --- python/nav/dhcp/kea_metrics.py | 68 +++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index b6ecec7048..8d93351aab 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -14,21 +14,23 @@ _logger = logging.getLogger(__name__) + class KeaDhcpMetricSource(DhcpMetricSource): """ Sends http requests to Kea Control Agent on `self.rest_uri` to fetch metrics from all subnets managed by the Kea DHCP server (serving ip version `dhcp_version` addresses) that the Kea Control Agent controls. """ + def __init__( - self, - address: str, - port: int, - *args, - https: bool = True, - dhcp_version: int = 4, - timeout = 10, - **kwargs, + self, + address: str, + port: int, + *args, + https: bool = True, + dhcp_version: int = 4, + timeout=10, + **kwargs, ): """ Instantiate a KeaDhcpMetricSource that fetches information via the Kea @@ -58,14 +60,16 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: subnets = _subnets_of_config(config, self.dhcp_version) for subnetid, prefix in subnets: for kea_key, nav_key in metric_keys: - kea_statisticname = f"subnet[{subnetid}].{kea_key}" - response = self._send_query(s, "statistic-get", name=kea_statisticname) - timeseries = response.get("arguments", {}).get(kea_statisticname, []) + kea_name = f"subnet[{subnetid}].{kea_key}" + response = self._send_query(s, "statistic-get", name=kea_name) + timeseries = response.get("arguments", {}).get(kea_name, []) if len(timeseries) == 0: _logger.error( "fetch_metrics: Could not fetch metric '%r' for subnet " "'%s' from Kea: '%s' from Kea is an empty list.", - nav_key, prefix, kea_statisticname, + nav_key, + prefix, + kea_name, ) for value, timestamp in timeseries: metrics.append( @@ -81,7 +85,6 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: return metrics - def _fetch_config(self, session: requests.Session) -> dict: """ Fetch the current config of the Kea DHCP server serving ip version @@ -92,7 +95,11 @@ def _fetch_config(self, session: requests.Session) -> dict: or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None or self._fetch_config_hash(session) != dhcp_confighash ): - self.dhcp_config = self._send_query(session, "config-get").get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) + self.dhcp_config = ( + self._send_query(session, "config-get") + .get("arguments", {}) + .get(f"Dhcp{self.dhcp_version}", None) + ) if self.dhcp_config is None: raise KeaError( "Could not fetch configuration of Kea DHCP server from Kea Control " @@ -101,14 +108,16 @@ def _fetch_config(self, session: requests.Session) -> dict: return self.dhcp_config - def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: """ Fetch the hash of the current config of the Kea DHCP server serving ip version `dhcp_version` """ - return self._send_query(session, "config-hash-get").get("arguments", {}).get("hash", None) - + return ( + self._send_query(session, "config-hash-get") + .get("arguments", {}) + .get("hash", None) + ) def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict: """ @@ -118,11 +127,13 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict exceptions are of type `KeaError`. Proper error responses as documented in the API are logged but results in an empty dictionary being returned. """ - postdata = json.dumps({ - "command": command, - "arguments": **kwargs, - "service": [f"dhcp{self.dhcp_version}"] - }) + postdata = json.dumps( + { + "command": command, + "arguments": {**kwargs}, + "service": [f"dhcp{self.dhcp_version}"], + } + ) _logger.info( "send_query: Post request to Kea Control Agent at %s with data %s", self.rest_uri, @@ -167,14 +178,16 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict "(responded with: %r) ", self.rest_uri, postdata, - responses + responses, ) return {} + def _parsetime(timestamp: str) -> int: """Parse the timestamp string used in Kea's timeseries into unix time""" return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) + def _subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP]]: """ List the id and prefix of subnets listed in the Kea DHCP configuration @@ -182,13 +195,14 @@ def _subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP] """ subnetkey = f"subnet{ip_version}" for subnet in chain.from_iterable( - [config.get(subnetkey, [])] - + [network.get(subnetkey, []) for network in config.get("shared-networks", [])] + [config.get(subnetkey, [])] + + [network.get(subnetkey, []) for network in config.get("shared-networks", [])] ): id = subnet.get("id", None) prefix = subnet.get("subnet", None) if id is None or prefix is None: - _logger.warning("subnets: id or prefix missing from a subnet's configuration: %r", subnet) + msg = "subnets: id or prefix missing from a subnet's configuration: %r" + _logger.warning(msg, subnet) continue yield id, IP(prefix) @@ -196,8 +210,10 @@ def _subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP] class KeaError(GeneralException): """Error related to interaction with a Kea Control Agent""" + class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" + # Successful operation. SUCCESS = 0 # General failure. From c6bfadadcaebb88007fd9fcf6d0df44a9004a86d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 11:40:16 +0200 Subject: [PATCH 45/77] Use nav.metrics.names.escape_metric_name() instead of manual escape --- python/nav/dhcp/generic_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index b427907dbc..5f7744026d 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -2,6 +2,7 @@ from enum import Enum from IPy import IP from nav.metrics import carbon +from nav.metrics.names import escape_metric_name from typing import Iterator @@ -42,10 +43,9 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: raise NotImplementedError def fetch_metrics_to_graphite(self, host, port): - fmt = str.maketrans({".": "_", "/": "_"}) # 192.0.2.0/24 --> 192_0_0_0_24 graphite_metrics = [] for metric in self.fetch_metrics(): - graphite_path = f"{self.graphite_prefix}.{str(metric.subnet_prefix).translate(fmt)}.{metric.key}" + graphite_path = f"{self.graphite_prefix}.{escape_metric_name(str(metric.subnet_prefix))}.{metric.key}" datapoint = (metric.timestamp, metric.value) graphite_metrics.append((graphite_path, datapoint)) carbon.send_metrics_to(graphite_metrics, host, port) From 79392cd456965baf8f9c39478a88e4fb989ae2c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 22 Jul 2024 14:21:35 +0200 Subject: [PATCH 46/77] Dump --- python/nav/dhcp/conv.bash | 3 + python/nav/dhcp/data.dat | 68 +++ tests/unittests/dhcp/kea_metrics_test.py | 535 +++++++---------------- 3 files changed, 229 insertions(+), 377 deletions(-) create mode 100644 python/nav/dhcp/conv.bash create mode 100644 python/nav/dhcp/data.dat diff --git a/python/nav/dhcp/conv.bash b/python/nav/dhcp/conv.bash new file mode 100644 index 0000000000..1f59c57030 --- /dev/null +++ b/python/nav/dhcp/conv.bash @@ -0,0 +1,3 @@ +while read -r line; do + date -d "$line" --iso-8601=ns +done < <(awk '/[0-9]+-[0-9]+-[0-9]+/ { gsub(/^[ ]*"/,"",$0); gsub(/"[ ]*$/,"",$0); print $0 }') diff --git a/python/nav/dhcp/data.dat b/python/nav/dhcp/data.dat new file mode 100644 index 0000000000..3fbf2ef67e --- /dev/null +++ b/python/nav/dhcp/data.dat @@ -0,0 +1,68 @@ +{ + "subnet[1].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140438" + ], + [ + 0, + "2024-07-05 20:44:54.230608" + ], + [ + 1, + "2024-07-05 09:15:05.626594" + ], + ], + "subnet[1].cumulative-assigned-addresses": [ + [ + 6, + "2024-07-22 09:06:58.140441" + ], + [ + 5, + "2024-07-05 09:15:05.626595" + ], + [ + 4, + "2024-07-04 09:18:47.679802" + ] + ], + "subnet[1].declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401071" + ] + ], + "subnet[1].reclaimed-declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401073" + ] + ], + "subnet[1].reclaimed-leases": [ + [ + 5, + "2024-07-05 20:44:54.230614" + ], + [ + 4, + "2024-07-04 16:29:36.612043" + ], + [ + 3, + "2024-07-04 14:22:18.181720" + ] + ], + "subnet[1].total-addresses": [ + [ + 239, + "2024-07-03 16:13:59.401058" + ] + ], + "subnet[1].v4-reservation-conflicts": [ + [ + 0, + "2024-07-03 16:13:59.401062" + ] + ], + } \ No newline at end of file diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index d79f0f6655..6b0d0168ae 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -8,245 +8,9 @@ import re from requests.exceptions import JSONDecodeError, HTTPError, Timeout -class TestSendQuery: - """Testing the list[KeaResponse] returned by send_query()""" - def test_response_with_success_status_should_succeed_and_be_logged(self, success_response, enqueue_post_response, testlog): - enqueue_post_response("command", success_response) - testlog.clear() - responses = send_dummy_query("command") - assert len(responses) == 4 - for response in responses: - assert response.success - assert isinstance(response.text, str) - assert isinstance(response.arguments, dict) - assert isinstance(response.service, str) - assert testlog.has_entries(logging.DEBUG, regexes=("query", "sen(d|t)", "192.0.2.2:80")) - - def test_response_with_error_status_should_succeed_and_be_logged(self, error_response, enqueue_post_response, testlog): - enqueue_post_response("command", error_response) - responses = send_dummy_query("command") - assert len(responses) == 5 - for response in responses: - assert not response.success - assert isinstance(response.text, str) - assert isinstance(response.arguments, dict) - assert isinstance(response.service, str) - assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) - assert testlog.has_entries(logging.DEBUG, n=1) - - def test_exceptions_should_be_logged_and_reraised_as_KeaError(self, enqueue_post_response, testlog): - enqueue_post_response("httperror", raiser(HTTPError)) - enqueue_post_response("timeout", raiser(Timeout)) - query = KeaQuery("httperror", [], {}) - with pytest.raises(KeaError): - responses = send_query(query, "192.0.2.2", 80) - assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) - assert testlog.has_entries(logging.DEBUG, regexes=("query", "fail|error|exception", "192.0.2.2:80")) - - testlog.clear() - - query = KeaQuery("timeout", [], {}) - with pytest.raises(KeaError): - responses = send_query(query, "192.0.2.2", 80) - assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) - assert testlog.has_entries(logging.DEBUG, regexes=("[^_]query", "fail|error|exception", "192.0.2.2:80")) - - def test_response_with_invalid_json_should_raise_and_be_logged(self, invalid_json_response, enqueue_post_response, testlog): - enqueue_post_response("command", invalid_json_response) - testlog.clear() - with pytest.raises(KeaError): - responses = send_dummy_query("command") - assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.2:80")) - assert testlog.has_entries(logging.DEBUG, regexes=("invalid", "json", "192.0.2.2:80")) - -class TestProcessingJsonIntoDataclass: - """ - Testing that json formatted Kea DHCP configuration is correctly processed - into python dataclasses - """ - def test_dhcp4_config_json_should_be_correctly_processed_into_KeaDhcpConfig( - self, - dhcp4_config, - dhcp4_config_with_shared_networks - ): - # Massaging and processing json with subnet4 list - j = json.loads(dhcp4_config) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 1 - subnet = config.subnets[0] - assert subnet.id == 1 - assert subnet.prefix == IP("192.0.0.0/8") - assert len(subnet.pools) == 2 - assert subnet.pools[0] == (IP("192.1.0.1"), IP("192.1.0.200")) - assert subnet.pools[1] == (IP("192.3.0.1"), IP("192.3.0.200")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - # Massaging and processing json with shared_networks list AND subnet4 list - j = json.loads(dhcp4_config_with_shared_networks) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 4 - subnets = {subnet.id: subnet for subnet in config.subnets} - - subnet1 = subnets[1] - assert subnet1.id == 1 - assert subnet1.prefix == IP("192.0.1.0/24") - assert len(subnet1.pools) == 1 - assert subnet1.pools[0] == (IP("192.0.1.1"), IP("192.0.1.200")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - subnet2 = subnets[2] - assert subnet2.id == 2 - assert subnet2.prefix == IP("192.0.2.0/24") - assert len(subnet2.pools) == 1 - assert subnet2.pools[0] == (IP("192.0.2.100"), IP("192.0.2.199")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - subnet3 = subnets[3] - assert subnet3.id == 3 - assert subnet3.prefix == IP("192.0.3.0/24") - assert len(subnet3.pools) == 1 - assert subnet3.pools[0] == (IP("192.0.3.100"), IP("192.0.3.199")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - subnet4 = subnets[4] - assert subnet4.id == 4 - assert subnet4.prefix == IP("10.0.0.0/8") - assert len(subnet4.pools) == 1 - assert subnet4.pools[0] == (IP("10.0.0.1"), IP("10.0.0.99")) - assert config.dhcp_version == 4 - assert config.config_hash is None - - def test_dhcp6_config_json_should_be_correctly_processed_into_KeaDhcpConfig(self, dhcp6_config): - j = json.loads(dhcp6_config) - config = KeaDhcpConfig.from_json(j) - assert len(config.subnets) == 2 - subnet1 = config.subnets[0] - assert subnet1.id == 1 - assert subnet1.prefix == IP("2001:db8:1:1::/64") - assert len(subnet1.pools) == 1 - assert subnet1.pools[0] == (IP("2001:db8:1:1::1"), IP("2001:db8:1:1::ffff")) - assert config.dhcp_version == 6 - assert config.config_hash is None - - subnet2 = config.subnets[1] - assert subnet2.id == 2 - assert subnet2.prefix == IP("2001:db8:1:2::/64") - assert len(subnet2.pools) == 2 - assert subnet2.pools[0] == (IP("2001:db8:1:2::1"), IP("2001:db8:1:2::ffff")) - assert subnet2.pools[1] == (IP("2001:db8:1:2::1:0"), IP("2001:db8:1:2::1:ffff")) - assert config.dhcp_version == 6 - assert config.config_hash is None - -""" -def test_invalid_json_response(testlog, invalid_json_response, enqueue_post_response): - enqueue_post_response("config-get", lambda **_: invalid_json_response) - enqueue_post_response("statistic-get", lambda **_: invalid_json_response) - testlog.clear() - source = KeaDhcpMetricSource(address="192.0.2.1", port=80) - with pytest.raises(KeaError): - source.fetch_and_set_dhcp_config() - assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") - - # fetch_dhcp_config_hash should not raise when the server does not support - # config-hash-get command - testlog.clear() - h = source.fetch_dhcp_config_hash() - assert h == None - assert testlog.has_entries( - logging.DEBUG, regex=".*no.*support.*hash.*|.*hash.*no.*support.*" - ) - - # fetch_dhcp_config_hash should raise when the server returns invalid - # json - enqueue_post_response("config-hash-get", lambda **_: invalid_json_response) - testlog.clear() - with pytest.raises(KeaError): - source.fetch_and_set_dhcp_config() - assert testlog.has_entries(logging.DEBUG, regex=".*invalid.*json.*") - - # FUNCTIONS USED EXTERNALLY SHOULD CATCH EXCEPTIONS AND LOG WARNINGS - # fetch_metrics is a method also used external to the module, and thus - # instead of raising it should log when the server returns invalid json - testlog.clear() - source.fetch_metrics() - assert testlog.has_entries(logging.WARNING, JSONDecodeError, n=1) -""" - -class TestKeaDhcpMetricSource: - def test_when_no_cached_KeaDhcpConfig_exist_should_fetch_and_set_correct_KeaDhcpConfig( - self, - dhcp4_config, - dhcp4_config_with_shared_networks, - dhcp6_config, - enqueue_post_response, - testlog, - ): - for config_string in dhcp4_config, dhcp6_config, dhcp4_config_with_shared_networks: - testlog.clear() - enqueue_post_response("config-get", response_json(config_string)) - source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) - assert source.kea_dhcp_config is None - config = source.fetch_and_set_dhcp_config() - actual_config = KeaDhcpConfig.from_json( - json.loads(config_string) - ) - assert config == actual_config - assert source.kea_dhcp_config == actual_config - assert testlog.has_entries(logging.DEBUG, regexes=("sen(d|t)[^:]", "[^_]query", "192.0.2.1")) - - def test_when_cached_KeaDhcpConfig_exist_and_local_hash_match_with_server_hash_should_not_fetch_new_KeaDhcpConfig( - self, - dhcp4_config, - dhcp4_config_with_shared_networks, - enqueue_post_response, - ): - enqueue_post_response("config-get", response_json(dhcp4_config)) - source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) - source.fetch_and_set_dhcp_config() - enqueue_post_response("config-hash-get", response_json(f'{{hash: "{source.kea_dhcp_config.config_hash}"}}')) - # The command 'config-hash-get' is set to return the same hash as is already cached. Thus, the test should fail if 'config-get' is queried. - enqueue_post_response("config-get", lambda **_: pytest.fail()) - source.fetch_and_set_dhcp_config() - - def test_when_cached_KeaDhcpConfig_exist_and_local_hash_doesnt_match_with_server_hash_should_fetch_new_KeaDhcpConfig( - self, - dhcp4_config, - dhcp4_config_with_shared_networks, - enqueue_post_response, - ): - enqueue_post_response("config-get", response_json(dhcp4_config)) - source = KeaDhcpMetricSource("192.0.2.1", 80, https=False) - source.fetch_and_set_dhcp_config() - assert source.kea_dhcp_config == KeaDhcpConfig.from_json(dhcp4_config) - - old_hash = source.kea_dhcp_config.config_hash - new_hash = "0" + old_hash[1:] if old_hash[0] != "0" else "1" + old_hash[1:] - - enqueue_post_response("config-hash-get", response_json(f'{{hash: "{new_hash}"}}')) - enqueue_post_response("config-get", response_json(dhcp4_config_with_shared_networks)) - source.fetch_and_set_dhcp_config() - assert source.kea_dhcp_config == KeaDhcpConfig.from_json(dhcp4_config_with_shared_networks) - - -# @pytest.fixture -# @enqueue_post_response -# def dhcp4_config_response_result_is_1(): -# return f''' -# {{ -# "result": 1, -# "arguments": {{ -# {DHCP4_CONFIG} -# }} -# }} -# ''' - -# def test_get_dhcp_config_result_is_1(dhcp4_config_result_is_1): -# with pytest.raises(Exception): # TODO: Change -# get_dhcp_server("example-org", dhcp_version=4) + +def test_should_return_all_metrics_from_normal_responses(): + @pytest.fixture @@ -290,22 +54,88 @@ def causes(e: BaseException): return n is None and len(entries) > 0 or len(entries) == n -def response_json(string): - return f''' - [ - {{ - "result": 0, - "arguments": {string} - }} - ] - ''' +def raiser(exception: type[Exception]): + def do_raise(*args, **kwargs): + raise exception + return do_raise + +@pytest.fixture(autouse=True) +def enqueue_post_response(monkeypatch): + """ + Any test that include this fixture, gets access to a function that + can be used to append text strings to a fifo queue of post + responses that in fifo order will be returned as proper Response + objects by calls to requests.post and requests.Session().post. + + This is how we mock what would otherwise be post requests to a + server. + """ + # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K + command_responses = {} + unknown_command_response = """[ + {{ + "result": 2, + "text": "'{0}' command not supported." + }} +]""" + + def new_post_function(url, *args, data="{}", **kwargs): + if isinstance(data, dict): + data = json.dumps(data) + elif isinstance(data, bytes): + data = data.decode("utf8") + if not isinstance(data, str): + pytest.fail( + f"data argument to the mocked requests.post() is of unknown type {type(data)}" + ) + + try: + data = json.loads(data) + command = data["command"] + except (JSONDecodeError, KeyError): + pytest.fail( + "All post requests that the Kea Control Agent receives from NAV" + "should be a JSON with a 'command' key. Instead, the mocked Kea " + f"Control Agent received {data!r}" + ) + fifo = command_responses.get(command, deque()) + if fifo: + first = fifo[0] + if callable(first): + text = first(arguments=data.get("arguments", {}), service=data.get("service", [])) + else: + text = str(first) + fifo.popleft() + else: + text = unknown_command_response.format(command) + response = requests.Response() + response._content = text.encode("utf8") + response.encoding = "utf8" + response.status_code = 400 + response.reason = "OK" + response.headers = kwargs.get("headers", {}) + response.cookies = kwargs.get("cookies", {}) + response.url = url + response.close = lambda: True + return response + + def new_post_method(self, url, *args, **kwargs): + return new_post_function(url, *args, **kwargs) + + def add_command_response(command_name, text): + command_responses.setdefault(command_name, deque()) + command_responses[command_name].append(text) + + monkeypatch.setattr(requests, 'post', new_post_function) + monkeypatch.setattr(requests.Session, 'post', new_post_method) + return add_command_response @pytest.fixture -def dhcp6_config(): - return '''{ +def DHCP6_CONFIG(): + config = { "Dhcp6": { "valid-lifetime": 4000, "renew-timer": 1000, @@ -346,11 +176,84 @@ def dhcp6_config(): } ] } -}''' + } + statistics = { + "subnet[1].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140438" + ], + [ + 0, + "2024-07-05 20:44:54.230608" + ], + [ + 1, + "2024-07-05 09:15:05.626594" + ], + ], + "subnet[1].cumulative-assigned-addresses": [ + [ + 6, + "2024-07-22 09:06:58.140441" + ], + [ + 5, + "2024-07-05 09:15:05.626595" + ], + [ + 4, + "2024-07-04 09:18:47.679802" + ] + ], + "subnet[1].declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401071" + ] + ], + "subnet[1].reclaimed-declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401073" + ] + ], + "subnet[1].reclaimed-leases": [ + [ + 5, + "2024-07-05 20:44:54.230614" + ], + [ + 4, + "2024-07-04 16:29:36.612043" + ], + [ + 3, + "2024-07-04 14:22:18.181720" + ] + ], + "subnet[1].total-addresses": [ + [ + 239, + "2024-07-03 16:13:59.401058" + ] + ], + "subnet[1].v4-reservation-conflicts": [ + [ + 0, + "2024-07-03 16:13:59.401062" + ] + ], + } -@pytest.fixture -def dhcp4_config(): - return ''' + metrics = [ + DhcpMetric("1970-01-01 01:00:01.000000000", + ] + + +DHCP6_CONFIG_STATISTICS = + +DHCP4_CONFIG = ''' { "Dhcp4": { "subnet4": [{ @@ -440,10 +343,7 @@ def dhcp4_config(): } ''' - -@pytest.fixture -def dhcp4_config_with_shared_networks(): - return '''{ +DHCP4_CONFIG_WITH_SHARED_NETWORKS = '''{ "Dhcp4": { "shared-networks": [ { @@ -483,122 +383,3 @@ def dhcp4_config_with_shared_networks(): }] } }''' - - -@pytest.fixture(autouse=True) -def enqueue_post_response(monkeypatch): - """ - Any test that include this fixture, gets access to a function that - can be used to append text strings to a fifo queue of post - responses that in fifo order will be returned as proper Response - objects by calls to requests.post and requests.Session().post. - - This is how we mock what would otherwise be post requests to a - server. - """ - # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K - command_responses = {} - unknown_command_response = """[ - {{ - "result": 2, - "text": "'{0}' command not supported." - }} -]""" - - def new_post_function(url, *args, data="{}", **kwargs): - if isinstance(data, dict): - data = json.dumps(data) - elif isinstance(data, bytes): - data = data.decode("utf8") - if not isinstance(data, str): - pytest.fail( - f"data argument to the mocked requests.post() is of unknown type {type(data)}" - ) - - try: - data = json.loads(data) - command = data["command"] - except (JSONDecodeError, KeyError): - pytest.fail( - "All post requests that the Kea Control Agent receives from NAV" - "should be a JSON with a 'command' key. Instead, the mocked Kea " - f"Control Agent received {data!r}" - ) - - fifo = command_responses.get(command, deque()) - if fifo: - first = fifo[0] - if callable(first): - text = first(arguments=data.get("arguments", {}), service=data.get("service", [])) - else: - text = str(first) - fifo.popleft() - else: - text = unknown_command_response.format(command) - - response = requests.Response() - response._content = text.encode("utf8") - response.encoding = "utf8" - response.status_code = 400 - response.reason = "OK" - response.headers = kwargs.get("headers", {}) - response.cookies = kwargs.get("cookies", {}) - response.url = url - response.close = lambda: True - return response - - def new_post_method(self, url, *args, **kwargs): - return new_post_function(url, *args, **kwargs) - - def add_command_response(command_name, text): - command_responses.setdefault(command_name, deque()) - command_responses[command_name].append(text) - - monkeypatch.setattr(requests, 'post', new_post_function) - monkeypatch.setattr(requests.Session, 'post', new_post_method) - - return add_command_response - - -@pytest.fixture -def success_response(): - return '''[ - {"result": 0, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 0, "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 0, "service": "d"}, - {"result": 0} - ]''' - - -@pytest.fixture -def error_response(): - return '''[ - {"result": 1, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 2, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 3, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 4, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"text": "b", "arguments": {"arg1": "val1"}, "service": "d"} - ]''' - - -@pytest.fixture -def invalid_json_response(): - return '''[ - {"result": 1, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 2, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 3, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"result": 4, "text": "b", "arguments": {"arg1": "val1"}, "service": "d"}, - {"text": "b", "arguments": {"arg1": "val1"}, "service": "d" - ]''' - -def send_dummy_query(command="command"): - return send_query( - query=KeaQuery(command, [], {}), - address="192.0.2.2", - port=80, - ) - -def raiser(exception: type[Exception]): - def do_raise(*args, **kwargs): - raise exception - return do_raise From fe196d3f4eac531074f38306a6ae93cc522ebc7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:14:36 +0200 Subject: [PATCH 47/77] Use full ipaddress string in graphite path ... that is, use IP.strNormal instead of str(IP) --- python/nav/dhcp/generic_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 5f7744026d..a16ff90a16 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -45,7 +45,7 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: def fetch_metrics_to_graphite(self, host, port): graphite_metrics = [] for metric in self.fetch_metrics(): - graphite_path = f"{self.graphite_prefix}.{escape_metric_name(str(metric.subnet_prefix))}.{metric.key}" + graphite_path = f"{self.graphite_prefix}.{escape_metric_name(metric.subnet_prefix.strNormal())}.{metric.key}" datapoint = (metric.timestamp, metric.value) graphite_metrics.append((graphite_path, datapoint)) carbon.send_metrics_to(graphite_metrics, host, port) From e6d75bd820a9227aca19313b4663228e48bfd073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:20:42 +0200 Subject: [PATCH 48/77] Make DhcpMetric hashable and immutable --- python/nav/dhcp/generic_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index a16ff90a16..1848c013b4 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -15,7 +15,7 @@ def __str__(self): return self.name.lower() # For use in graphite path -@dataclass +@dataclass(frozen=True) class DhcpMetric: timestamp: int subnet_prefix: IP From 8dacb957ca4ce8d74b9b9c394827a5fa63b39b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:22:44 +0200 Subject: [PATCH 49/77] Rewrite tests --- tests/unittests/dhcp/kea_metrics_test.py | 864 +++++++++++++++-------- 1 file changed, 565 insertions(+), 299 deletions(-) diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 6b0d0168ae..858e6d28c8 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -1,5 +1,6 @@ from collections import deque from nav.dhcp.kea_metrics import * +from nav.dhcp.generic_metrics import DhcpMetric import pytest import requests from IPy import IP @@ -7,68 +8,559 @@ import logging import re from requests.exceptions import JSONDecodeError, HTTPError, Timeout +from datetime import timezone + +def test_dhcp6_config_and_statistic_response_that_is_valid_should_return_every_metric(valid_dhcp6, responsequeue): + config, statistics, expected_metrics = valid_dhcp6 + responsequeue.prefill("dhcp6", config, statistics) + source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=6, tzinfo=timezone.utc) + assert set(source.fetch_metrics()) == set(expected_metrics) + +def test_dhcp4_config_and_statistic_response_that_is_valid_should_return_every_metric(valid_dhcp4, responsequeue): + config, statistics, expected_metrics = valid_dhcp4 + responsequeue.prefill("dhcp4", config, statistics) + source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4, tzinfo=timezone.utc) + assert set(source.fetch_metrics()) == set(expected_metrics) + +@pytest.mark.parametrize("status", [status for status in KeaStatus if status != KeaStatus.SUCCESS]) +def test_config_response_with_error_status_should_raise_KeaError(valid_dhcp4, responsequeue, status): + """ + If Kea responds with an error while fetching the Kea DHCP's config during + fetch_metrics(), we cannot continue + """ + config, statistics, _ = valid_dhcp4 + responsequeue.prefill("dhcp4", None, statistics) + responsequeue.add("config-get", kearesponse(config, status=status)) + source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4) + with pytest.raises(KeaError): + source.fetch_metrics() + +def test_any_response_with_invalid_format_should_raise_KeaError(valid_dhcp4, responsequeue): + """ + If Kea responds with an invalid format (i.e. in an unrecognizable way), we + should fail loudly, because proboably either the location we're re + requesting is not a Kea Control Agent, or there's a part of the API that + we've not covered. + """ + config, statistics, _ = valid_dhcp4 + source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4) + + responsequeue.add("config-get", "{}") + with pytest.raises(KeaError): + source.fetch_metrics() + + responsequeue.clear() + + responsequeue.prefill("dhcp4", config, None) + responsequeue.add("statistic-get", "{}") + with pytest.raises(KeaError): + source.fetch_metrics() + responsequeue.clear() -def test_should_return_all_metrics_from_normal_responses(): + responsequeue.prefill("dhcp4", None, statistics) + responsequeue.add("config-get", "{}") + with pytest.raises(KeaError): + source.fetch_metrics() + responsequeue.clear() + + # config-hash-get is only called if some config-get includes a hash we can compare + # with the next time we're attempting to fetch a config: + config["Dhcp4"]["hash"] = "foo" + responsequeue.prefill("dhcp4", config, statistics) + responsequeue.add("config-hash-get", "{}") + with pytest.raises(KeaError): + source.fetch_metrics() + + +def test_all_responses_is_empty_but_valid_should_yield_no_metrics(valid_dhcp4, responsequeue): + """ + If the Kea DHCP server we query does not have any subnets configured, the + correct thing to do is to return an empty iterable, (as opposed to failing). + + Likewise, if it returns no statistics for its configured subnets, the + correct thing to do is to return an empty iterable. + """ + config, statistics, _ = valid_dhcp4 + responsequeue.prefill("dhcp4", None, statistics) + responsequeue.add("config-get", lambda **_: kearesponse({"Dhcp4": {}})) + source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4, tzinfo=timezone.utc) + assert list(source.fetch_metrics()) == [] + + responsequeue.clear() + + responsequeue.prefill("dhcp4", config, None) + responsequeue.add("statistic-get", lambda arguments, **_: kearesponse({arguments["name"]: []})) + assert list(source.fetch_metrics()) == [] + + responsequeue.clear() + + responsequeue.prefill("dhcp4", config, None) + responsequeue.add("statistic-get", lambda **_: kearesponse({})) + assert list(source.fetch_metrics()) == [] @pytest.fixture -def testlog(caplog): - caplog.clear() - caplog.set_level(logging.DEBUG) - return LogChecker(caplog) - - -class LogChecker: - def __init__(self, caplog): - self.caplog = caplog - - def clear(self): - self.caplog.clear() - - def has_entries(self, level, exception=None, regexes=None, n=None): - """ - Check if there is any log entries of logging level `level`, optionally - made for an exception `exception`, optionally with all regexes in - `regexes` fully matching some substring of the log message, and - optionally requiring that there is exactly `n` such records logged. - """ - - def causes(e: BaseException): - while e: - yield type(e) - e = e.__cause__ - - entries = [ - entry - for entry in self.caplog.records - if entry.levelno >= level - and ( - exception is None - or entry.exc_info is not None - and exception in causes(entry.exc_info[1]) - ) - and (regexes is None or all(re.search(regex, entry.message.lower(), re.DOTALL) for regex in regexes)) - ] - return n is None and len(entries) > 0 or len(entries) == n +def valid_dhcp6(): + config = { + "Dhcp6": { + "valid-lifetime": 4000, + "renew-timer": 1000, + "rebind-timer": 2000, + "preferred-lifetime": 3000, + + "interfaces-config": { + "interfaces": [ "eth0" ] + }, + + "lease-database": { + "type": "memfile", + "persist": True, + "name": "/var/lib/kea/dhcp6.leases" + }, + + "subnet6": [ + { + "id": 1, + "subnet": "2001:db8:1:1::/64", + "pools": [ + { + "pool": "2001:db8:1:1::1-2001:db8:1:1::ffff" + } + ] + }, + { + "id": 2, + "subnet": "2001:db8:1:2::/64", + "pools": [ + { + "pool": "2001:db8:1:2::1-2001:db8:1:2::ffff" + }, + { + "pool": "2001:db8:1:2::1:0/112" + } + ] + } + ], + "shared-networks": [ + { + "name": "shared-network-1", + "subnet6": [ + { + "id": 3, + "subnet": "2001:db8:1:3::/64", + }, + { + "id": 4, + "subnet": "2001:db8:1:4::/64", + } + ] + }, + { + "name": "shared-network-2", + "subnet6": [ + { + "id": 5, + "subnet": "2001:db8:1:5::/64", + } + ] + } + ], + } + } + statistics = { + "subnet[1].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140438" + ], + [ + 0, + "2024-07-05 20:44:54.230608" + ], + [ + 1, + "2024-07-05 09:15:05.626594" + ], + ], + "subnet[1].declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401071" + ] + ], + "subnet[1].total-addresses": [ + [ + 239, + "2024-07-03 16:13:59.401058" + ] + ], + "subnet[2].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140439" + ], + [ + 1, + "2024-07-05 20:44:54.230609" + ], + [ + 2, + "2024-07-05 09:15:05.626595" + ], + ], + "subnet[2].declined-addresses": [ + [ + 1, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[2].total-addresses": [ + [ + 240, + "2024-07-03 16:13:59.401059" + ] + ], + "subnet[3].assigned-addresses": [ + [ + 4, + "2024-07-22 09:06:58.140439" + ], + [ + 5, + "2024-07-05 20:44:54.230609" + ], + ], + "subnet[3].declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[3].total-addresses": [ + [ + 241, + "2024-07-03 16:13:59.401059" + ] + ], + "subnet[4].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140439" + ], + [ + 1, + "2024-07-05 20:44:54.230609" + ], + ], + "subnet[4].declined-addresses": [ + [ + 1, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[4].total-addresses": [ + [ + 242, + "2024-07-03 16:13:59.401059" + ] + ], + "subnet[5].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140439" + ], + [ + 1, + "2024-07-05 20:44:54.230609" + ], + ], + "subnet[5].declined-addresses": [ + [ + 1, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[5].total-addresses": [ + [ + 243, + "2024-07-03 16:13:59.401059" + ] + ], + + } + + expected_metrics = [ + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.CUR, 0), + DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.TOUCH, 0), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.MAX, 239), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.CUR, 2), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.TOUCH, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.MAX, 240), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.CUR, 4), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.CUR, 5), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.TOUCH, 0), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.MAX, 241), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.TOUCH, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.MAX, 242), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.TOUCH, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.MAX, 243), + ] + + return config, statistics, expected_metrics + +@pytest.fixture +def valid_dhcp4(): + config = { + "Dhcp4": { + "valid-lifetime": 4000, + "renew-timer": 1000, + "rebind-timer": 2000, + "preferred-lifetime": 3000, + + "interfaces-config": { + "interfaces": [ "eth0" ] + }, + + "lease-database": { + "type": "memfile", + "persist": True, + "name": "/var/lib/kea/dhcp6.leases" + }, + + "subnet4": [ + { + "id": 1, + "subnet": "192.0.1.0/24", + "pools": [ + { + "pool": "192.0.1.1-192.0.1.10" + } + ] + }, + { + "id": 2, + "subnet": "192.0.2.0/24", + "pools": [ + { + "pool": "192.0.2.1-192.0.2.10" + }, + { + "pool": "192.0.2.128/25" + } + ] + } + ], + "shared-networks": [ + { + "name": "shared-network-1", + "subnet4": [ + { + "id": 3, + "subnet": "192.0.3.0/24", + }, + { + "id": 4, + "subnet": "192.0.4.0/24", + } + ] + }, + { + "name": "shared-network-2", + "subnet4": [ + { + "id": 5, + "subnet": "192.0.5.0/24", + } + ] + } + ], + } + } + statistics = { + "subnet[1].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140438" + ], + [ + 0, + "2024-07-05 20:44:54.230608" + ], + [ + 1, + "2024-07-05 09:15:05.626594" + ], + ], + "subnet[1].declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401071" + ] + ], + "subnet[1].total-addresses": [ + [ + 239, + "2024-07-03 16:13:59.401058" + ] + ], + "subnet[2].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140439" + ], + [ + 1, + "2024-07-05 20:44:54.230609" + ], + [ + 2, + "2024-07-05 09:15:05.626595" + ], + ], + "subnet[2].declined-addresses": [ + [ + 1, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[2].total-addresses": [ + [ + 240, + "2024-07-03 16:13:59.401059" + ] + ], + "subnet[3].assigned-addresses": [ + [ + 4, + "2024-07-22 09:06:58.140439" + ], + [ + 5, + "2024-07-05 20:44:54.230609" + ], + ], + "subnet[3].declined-addresses": [ + [ + 0, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[3].total-addresses": [ + [ + 241, + "2024-07-03 16:13:59.401059" + ] + ], + "subnet[4].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140439" + ], + [ + 1, + "2024-07-05 20:44:54.230609" + ], + ], + "subnet[4].declined-addresses": [ + [ + 1, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[4].total-addresses": [ + [ + 242, + "2024-07-03 16:13:59.401059" + ] + ], + "subnet[5].assigned-addresses": [ + [ + 1, + "2024-07-22 09:06:58.140439" + ], + [ + 1, + "2024-07-05 20:44:54.230609" + ], + ], + "subnet[5].declined-addresses": [ + [ + 1, + "2024-07-03 16:13:59.401072" + ] + ], + "subnet[5].total-addresses": [ + [ + 243, + "2024-07-03 16:13:59.401059" + ] + ], + + } + + expected_metrics = [ + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.CUR, 0), + DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.TOUCH, 0), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.MAX, 239), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.CUR, 2), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.TOUCH, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.MAX, 240), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.CUR, 4), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.CUR, 5), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.TOUCH, 0), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.MAX, 241), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.TOUCH, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.MAX, 242), + DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.CUR, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.TOUCH, 1), + DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.MAX, 243), + ] + + return config, statistics, expected_metrics +def kearesponse(val, status=KeaStatus.SUCCESS): + return f''' +[ + {{ + "result": {status}, + "arguments": {json.dumps(val)} + }} +] + ''' -def raiser(exception: type[Exception]): - def do_raise(*args, **kwargs): - raise exception - return do_raise @pytest.fixture(autouse=True) -def enqueue_post_response(monkeypatch): +def responsequeue(monkeypatch): """ - Any test that include this fixture, gets access to a function that - can be used to append text strings to a fifo queue of post - responses that in fifo order will be returned as proper Response - objects by calls to requests.post and requests.Session().post. + Any test that include this fixture, will automatically mock + requests.Session.post() and requests.post(). The fixture returns a + namespace with three functions: + + responsequeue.add() can be used to append text strings or functions that + return text strings to a fifo queue of post responses that in fifo order + will be returned as proper requests.Response objects on calls to + requests.post() and requests.Session().post(). - This is how we mock what would otherwise be post requests to a - server. + responsequeue.remove() removes a specific fifo queue. + + responsequeue.clear() can be used to clear the all fifo queues. """ # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K command_responses = {} @@ -128,258 +620,32 @@ def add_command_response(command_name, text): command_responses.setdefault(command_name, deque()) command_responses[command_name].append(text) - monkeypatch.setattr(requests, 'post', new_post_function) - monkeypatch.setattr(requests.Session, 'post', new_post_method) - - return add_command_response - -@pytest.fixture -def DHCP6_CONFIG(): - config = { -"Dhcp6": { - "valid-lifetime": 4000, - "renew-timer": 1000, - "rebind-timer": 2000, - "preferred-lifetime": 3000, - - "interfaces-config": { - "interfaces": [ "eth0" ] - }, + def remove_command_responses(command_name): + command_responses.pop(command_name, None) - "lease-database": { - "type": "memfile", - "persist": true, - "name": "/var/lib/kea/dhcp6.leases" - }, + def clear_command_responses(): + command_responses.clear() - "subnet6": [ - { - "id": 1, - "subnet": "2001:db8:1:1::/64", - "pools": [ - { - "pool": "2001:db8:1:1::1-2001:db8:1:1::ffff" - } - ] - }, - { - "id": 2, - "subnet": "2001:db8:1:2::/64", - "pools": [ - { - "pool": "2001:db8:1:2::1-2001:db8:1:2::ffff" - }, - { - "pool": "2001:db8:1:2::1:0/112" - } - ] - } - ] -} - } - statistics = { - "subnet[1].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140438" - ], - [ - 0, - "2024-07-05 20:44:54.230608" - ], - [ - 1, - "2024-07-05 09:15:05.626594" - ], - ], - "subnet[1].cumulative-assigned-addresses": [ - [ - 6, - "2024-07-22 09:06:58.140441" - ], - [ - 5, - "2024-07-05 09:15:05.626595" - ], - [ - 4, - "2024-07-04 09:18:47.679802" - ] - ], - "subnet[1].declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401071" - ] - ], - "subnet[1].reclaimed-declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401073" - ] - ], - "subnet[1].reclaimed-leases": [ - [ - 5, - "2024-07-05 20:44:54.230614" - ], - [ - 4, - "2024-07-04 16:29:36.612043" - ], - [ - 3, - "2024-07-04 14:22:18.181720" - ] - ], - "subnet[1].total-addresses": [ - [ - 239, - "2024-07-03 16:13:59.401058" - ] - ], - "subnet[1].v4-reservation-conflicts": [ - [ - 0, - "2024-07-03 16:13:59.401062" - ] - ], - } + def prefill_command_responses(expected_service, config=None, statistics=None): + def config_get_response(arguments, service): + assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + return kearesponse(config) + def statistic_get_response(arguments, service): + assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + return kearesponse({arguments["name"]: statistics[arguments["name"]]}) - metrics = [ - DhcpMetric("1970-01-01 01:00:01.000000000", - ] + if config is not None: + add_command_response("config-get", config_get_response) + if statistics is not None: + add_command_response("statistic-get", statistic_get_response) + class ResponseQueue: + add = add_command_response + remove = remove_command_responses + clear = clear_command_responses + prefill = prefill_command_responses -DHCP6_CONFIG_STATISTICS = + monkeypatch.setattr(requests, 'post', new_post_function) + monkeypatch.setattr(requests.Session, 'post', new_post_method) -DHCP4_CONFIG = ''' - { - "Dhcp4": { - "subnet4": [{ - "4o6-interface": "eth1", - "4o6-interface-id": "ethx", - "4o6-subnet": "2001:db8:1:1::/64", - "allocator": "iterative", - "authoritative": false, - "boot-file-name": "/tmp/boot", - "client-class": "foobar", - "ddns-generated-prefix": "myhost", - "ddns-override-client-update": true, - "ddns-override-no-update": true, - "ddns-qualifying-suffix": "example.org", - "ddns-replace-client-name": "never", - "ddns-send-updates": true, - "ddns-update-on-renew": true, - "ddns-use-conflict-resolution": true, - "hostname-char-replacement": "x", - "hostname-char-set": "[^A-Za-z0-9.-]", - "id": 1, - "interface": "eth0", - "match-client-id": true, - "next-server": "0.0.0.0", - "store-extended-info": true, - "option-data": [ - { - "always-send": true, - "code": 3, - "csv-format": true, - "data": "192.0.3.1", - "name": "routers", - "space": "dhcp4" - } - ], - "pools": [ - { - "client-class": "phones_server1", - "option-data": [], - "pool": "192.1.0.1 - 192.1.0.200", - "pool-id": 7, - "require-client-classes": [ "late" ] - }, - { - "client-class": "phones_server2", - "option-data": [], - "pool": "192.3.0.1 - 192.3.0.200", - "require-client-classes": [] - } - ], - "rebind-timer": 40, - "relay": { - "ip-addresses": [ - "192.168.56.1" - ] - }, - "renew-timer": 30, - "reservations-global": true, - "reservations-in-subnet": true, - "reservations-out-of-pool": true, - "calculate-tee-times": true, - "t1-percent": 0.5, - "t2-percent": 0.75, - "cache-threshold": 0.25, - "cache-max-age": 1000, - "reservations": [ - { - "circuit-id": "01:11:22:33:44:55:66", - "ip-address": "192.0.2.204", - "hostname": "foo.example.org", - "option-data": [ - { - "name": "vivso-suboptions", - "data": "4491" - } - ] - } - ], - "require-client-classes": [ "late" ], - "server-hostname": "myhost.example.org", - "subnet": "192.0.0.0/8", - "valid-lifetime": 6000, - "min-valid-lifetime": 4000, - "max-valid-lifetime": 8000 - }] - } - } -''' - -DHCP4_CONFIG_WITH_SHARED_NETWORKS = '''{ - "Dhcp4": { - "shared-networks": [ - { - "name": "shared-network-1", - "subnet4": [ - { - "id": 4, - "subnet": "10.0.0.0/8", - "pools": [ { "pool": "10.0.0.1 - 10.0.0.99" } ] - }, - { - "id": 3, - "subnet": "192.0.3.0/24", - "pools": [ { "pool": "192.0.3.100 - 192.0.3.199" } ] - } - ] - }, - { - "name": "shared-network-2", - "subnet4": [ - { - "id": 2, - "subnet": "192.0.2.0/24", - "pools": [ { "pool": "192.0.2.100 - 192.0.2.199" } ] - } - ] - } - ], - "subnet4": [{ - "id": 1, - "subnet": "192.0.1.0/24", - "pools": [ - { - "pool": "192.0.1.1 - 192.0.1.200" - } - ] - }] - } - }''' + return ResponseQueue From 65a49412e3873293c64dba6911ee4510bf8293a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:24:18 +0200 Subject: [PATCH 50/77] Fix constant modified-config-during-metric-fetch error logging --- python/nav/dhcp/kea_metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 8d93351aab..143778aa71 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -188,11 +188,12 @@ def _parsetime(timestamp: str) -> int: return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) -def _subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP]]: +def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: """ List the id and prefix of subnets listed in the Kea DHCP configuration `config` """ + subnets = [] subnetkey = f"subnet{ip_version}" for subnet in chain.from_iterable( [config.get(subnetkey, [])] @@ -204,7 +205,8 @@ def _subnets_of_config(config: dict, ip_version: int) -> Iterator[tuple[int, IP] msg = "subnets: id or prefix missing from a subnet's configuration: %r" _logger.warning(msg, subnet) continue - yield id, IP(prefix) + subnets.append((id, IP(prefix))) + return subnets class KeaError(GeneralException): From 7181241d747c1e772f25049f5e04840d33808095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:30:34 +0200 Subject: [PATCH 51/77] Use datetime instances with timezone data instead of unix timestamps. Why: Datetime instances are the most precise, and can easily be converted to unix timestamps is need be. Datetime instances also makes it easy to work with timezone differences, which we sadly seem to have to care about since the Kea Control Agent doesn't provide timezone data along with its timestamps. --- python/nav/dhcp/kea_metrics.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 143778aa71..4fda757350 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -29,18 +29,26 @@ def __init__( *args, https: bool = True, dhcp_version: int = 4, - timeout=10, + timeout: int = 10, + tzinfo: datetime.tzinfo = datetime.now().astimezone().tzinfo, **kwargs, ): """ Instantiate a KeaDhcpMetricSource that fetches information via the Kea Control Agent listening to `port` on `address`. + + :param https: if True, use https. Otherwise, use http + :param dhcp_version: ip version served by Kea DHCP server + :param timeout: how long to wait for http response from Kea Control Agent before timing out + :param tzinfo: the timezone of the Kea Control Agent. We must specify its timezone explicitly because timestamps it responds with bear no timezone information. If this parameter is not given, we assume that the the Kea Control Agent and this machine is configured to use the same timezone. """ super(*args, **kwargs) scheme = "https" if https else "http" self.rest_uri = f"{scheme}://{address}:{port}/" self.dhcp_version = dhcp_version self.dhcp_config: Optional[dict] = None + self.timeout = timeout + self.tzinfo = tzinfo def fetch_metrics(self) -> Iterator[DhcpMetric]: """ @@ -182,10 +190,9 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict ) return {} - -def _parsetime(timestamp: str) -> int: - """Parse the timestamp string used in Kea's timeseries into unix time""" - return calendar.timegm(time.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")) + def _parsetime(self, timestamp: str) -> int: + """Parse the timestamp string used in Kea's timeseries into unix time""" + return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=self.tzinfo) def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: From 4f7a32c4582619177cc8c0e4aea552085e8424e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:34:10 +0200 Subject: [PATCH 52/77] Fail if status of a config's response is not SUCCESS Why: If we don't obtain the config, we do not have enough information to start fetching subnet metrics; in this case there is no way to move forward and there's no hope of obtaining some useful data by continuing the call. --- python/nav/dhcp/kea_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 4fda757350..b832b06431 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -108,7 +108,7 @@ def _fetch_config(self, session: requests.Session) -> dict: .get("arguments", {}) .get(f"Dhcp{self.dhcp_version}", None) ) - if self.dhcp_config is None: + if self.dhcp_config is None or status != KeaStatus.SUCCESS: raise KeaError( "Could not fetch configuration of Kea DHCP server from Kea Control " f"Agent at {self.rest_uri}" From cac787db6c42b78af66058ac0cbe23da4971e469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 15:38:48 +0200 Subject: [PATCH 53/77] Rename variables and tidy up code --- python/nav/dhcp/kea_metrics.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index b832b06431..5e6a0ef7f6 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -4,6 +4,7 @@ from nav.dhcp.generic_metrics import DhcpMetricSource from nav.errors import GeneralException from nav.dhcp.generic_metrics import DhcpMetric, DhcpMetricKey, DhcpMetricSource +from datetime import datetime import logging from requests import RequestException, JSONDecodeError import requests @@ -63,7 +64,7 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: ) metrics = [] - with requests.Session as s: + with requests.Session() as s: config = self._fetch_config(s) subnets = _subnets_of_config(config, self.dhcp_version) for subnetid, prefix in subnets: @@ -81,10 +82,11 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: ) for value, timestamp in timeseries: metrics.append( - DhcpMetric(_parsetime(timestamp), prefix, nav_key, value) + DhcpMetric(self._parsetime(timestamp), prefix, nav_key, value) ) - if sorted(subnets) != sorted(_subnets_of_config(self._fetch_config())): + newsubnets = _subnets_of_config(self._fetch_config(s), self.dhcp_version) + if sorted(subnets) != sorted(newsubnets): _logger.error( "Subnet configuration was modified during metric fetching, " "this may cause metric data being associated with wrong " @@ -103,10 +105,10 @@ def _fetch_config(self, session: requests.Session) -> dict: or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None or self._fetch_config_hash(session) != dhcp_confighash ): + response = self._send_query(session, "config-get") + status = response.get("result", KeaStatus.ERROR) self.dhcp_config = ( - self._send_query(session, "config-get") - .get("arguments", {}) - .get(f"Dhcp{self.dhcp_version}", None) + response.get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) ) if self.dhcp_config is None or status != KeaStatus.SUCCESS: raise KeaError( @@ -151,7 +153,6 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict responses = session.post( self.rest_uri, data=postdata, - headers=self.rest_headers, timeout=self.timeout, ) responses = responses.json() @@ -164,6 +165,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict f"Uri {self.rest_uri} most likely not pointing at a Kea " f"Control Agent (expected json, responded with: {responses!r})", ) from err + if not isinstance(responses, list): # See https://kea.readthedocs.io/en/kea-2.6.0/arm/ctrl-channel.html#control-agent-command-response-format raise KeaError( From 3eafeb62446b3025f85920aa71dbcacdd8c13e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 16:08:29 +0200 Subject: [PATCH 54/77] Adjust documentation --- python/nav/dhcp/kea_metrics.py | 10 ++++++---- tests/unittests/dhcp/kea_metrics_test.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 5e6a0ef7f6..9ac672f3ea 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -41,7 +41,7 @@ def __init__( :param https: if True, use https. Otherwise, use http :param dhcp_version: ip version served by Kea DHCP server :param timeout: how long to wait for http response from Kea Control Agent before timing out - :param tzinfo: the timezone of the Kea Control Agent. We must specify its timezone explicitly because timestamps it responds with bear no timezone information. If this parameter is not given, we assume that the the Kea Control Agent and this machine is configured to use the same timezone. + :param tzinfo: the timezone of the Kea Control Agent. """ super(*args, **kwargs) scheme = "https" if https else "http" @@ -133,9 +133,11 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict """ Send `command` to the Kea Control Agent. An exception is raised iff there was an HTTP related error while sending `command` or a response - does not look like it is coming from a Kea Control Agent. All raised - exceptions are of type `KeaError`. Proper error responses as documented - in the API are logged but results in an empty dictionary being returned. + does not look like it is coming from a Kea Control Agent or if the + request likely was rejected by the Kea Control Agent. All raised + exceptions are of type `KeaError`. Valid Kea error responses such as + those with result == KeaStatus.ERROR are only logged as an error, and + result in an empty dictionary being returned. """ postdata = json.dumps( { diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 858e6d28c8..440a1b498a 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -26,7 +26,7 @@ def test_dhcp4_config_and_statistic_response_that_is_valid_should_return_every_m def test_config_response_with_error_status_should_raise_KeaError(valid_dhcp4, responsequeue, status): """ If Kea responds with an error while fetching the Kea DHCP's config during - fetch_metrics(), we cannot continue + fetch_metrics(), we cannot continue further, so we fail. """ config, statistics, _ = valid_dhcp4 responsequeue.prefill("dhcp4", None, statistics) @@ -38,8 +38,8 @@ def test_config_response_with_error_status_should_raise_KeaError(valid_dhcp4, re def test_any_response_with_invalid_format_should_raise_KeaError(valid_dhcp4, responsequeue): """ If Kea responds with an invalid format (i.e. in an unrecognizable way), we - should fail loudly, because proboably either the location we're re - requesting is not a Kea Control Agent, or there's a part of the API that + should fail loudly, because chances are either the host we're sending + requests to is not a Kea Control Agent, or there's a part of the API that we've not covered. """ config, statistics, _ = valid_dhcp4 From 92f54e15ae58b88db63aaf030ebff19ba6d63d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Tue, 23 Jul 2024 16:45:21 +0200 Subject: [PATCH 55/77] Fix linting errors --- python/nav/dhcp/conv.bash | 3 - python/nav/dhcp/data.dat | 68 -- python/nav/dhcp/kea_metrics.py | 26 +- tests/unittests/dhcp/kea_metrics_test.py | 799 ++++++++++++----------- 4 files changed, 435 insertions(+), 461 deletions(-) delete mode 100644 python/nav/dhcp/conv.bash delete mode 100644 python/nav/dhcp/data.dat diff --git a/python/nav/dhcp/conv.bash b/python/nav/dhcp/conv.bash deleted file mode 100644 index 1f59c57030..0000000000 --- a/python/nav/dhcp/conv.bash +++ /dev/null @@ -1,3 +0,0 @@ -while read -r line; do - date -d "$line" --iso-8601=ns -done < <(awk '/[0-9]+-[0-9]+-[0-9]+/ { gsub(/^[ ]*"/,"",$0); gsub(/"[ ]*$/,"",$0); print $0 }') diff --git a/python/nav/dhcp/data.dat b/python/nav/dhcp/data.dat deleted file mode 100644 index 3fbf2ef67e..0000000000 --- a/python/nav/dhcp/data.dat +++ /dev/null @@ -1,68 +0,0 @@ -{ - "subnet[1].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140438" - ], - [ - 0, - "2024-07-05 20:44:54.230608" - ], - [ - 1, - "2024-07-05 09:15:05.626594" - ], - ], - "subnet[1].cumulative-assigned-addresses": [ - [ - 6, - "2024-07-22 09:06:58.140441" - ], - [ - 5, - "2024-07-05 09:15:05.626595" - ], - [ - 4, - "2024-07-04 09:18:47.679802" - ] - ], - "subnet[1].declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401071" - ] - ], - "subnet[1].reclaimed-declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401073" - ] - ], - "subnet[1].reclaimed-leases": [ - [ - 5, - "2024-07-05 20:44:54.230614" - ], - [ - 4, - "2024-07-04 16:29:36.612043" - ], - [ - 3, - "2024-07-04 14:22:18.181720" - ] - ], - "subnet[1].total-addresses": [ - [ - 239, - "2024-07-03 16:13:59.401058" - ] - ], - "subnet[1].v4-reservation-conflicts": [ - [ - 0, - "2024-07-03 16:13:59.401062" - ] - ], - } \ No newline at end of file diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 9ac672f3ea..2b5e5e413d 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -67,7 +67,7 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: with requests.Session() as s: config = self._fetch_config(s) subnets = _subnets_of_config(config, self.dhcp_version) - for subnetid, prefix in subnets: + for subnetid, netprefix in subnets: for kea_key, nav_key in metric_keys: kea_name = f"subnet[{subnetid}].{kea_key}" response = self._send_query(s, "statistic-get", name=kea_name) @@ -77,13 +77,12 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: "fetch_metrics: Could not fetch metric '%r' for subnet " "'%s' from Kea: '%s' from Kea is an empty list.", nav_key, - prefix, + netprefix, kea_name, ) - for value, timestamp in timeseries: - metrics.append( - DhcpMetric(self._parsetime(timestamp), prefix, nav_key, value) - ) + for val, t in timeseries: + metric = DhcpMetric(self._parsetime(t), netprefix, nav_key, val) + metrics.append(metric) newsubnets = _subnets_of_config(self._fetch_config(s), self.dhcp_version) if sorted(subnets) != sorted(newsubnets): @@ -107,15 +106,13 @@ def _fetch_config(self, session: requests.Session) -> dict: ): response = self._send_query(session, "config-get") status = response.get("result", KeaStatus.ERROR) - self.dhcp_config = ( - response.get("arguments", {}).get(f"Dhcp{self.dhcp_version}", None) - ) + arguments = response.get("arguments", {}) + self.dhcp_config = arguments.get(f"Dhcp{self.dhcp_version}", None) if self.dhcp_config is None or status != KeaStatus.SUCCESS: raise KeaError( "Could not fetch configuration of Kea DHCP server from Kea Control " f"Agent at {self.rest_uri}" ) - return self.dhcp_config def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: @@ -135,9 +132,9 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict there was an HTTP related error while sending `command` or a response does not look like it is coming from a Kea Control Agent or if the request likely was rejected by the Kea Control Agent. All raised - exceptions are of type `KeaError`. Valid Kea error responses such as - those with result == KeaStatus.ERROR are only logged as an error, and - result in an empty dictionary being returned. + exceptions are of type `KeaError`. Occurrences of valid Kea error + responses such as those with result == KeaStatus.ERROR are logged as an + error, and result in an empty dictionary being returned. """ postdata = json.dumps( { @@ -196,7 +193,8 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict def _parsetime(self, timestamp: str) -> int: """Parse the timestamp string used in Kea's timeseries into unix time""" - return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=self.tzinfo) + fmt = "%Y-%m-%d %H:%M:%S.%f" + return datetime.strptime(timestamp, fmt).replace(tzinfo=self.tzinfo) def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 440a1b498a..c232b89d35 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -10,20 +10,31 @@ from requests.exceptions import JSONDecodeError, HTTPError, Timeout from datetime import timezone -def test_dhcp6_config_and_statistic_response_that_is_valid_should_return_every_metric(valid_dhcp6, responsequeue): + +def test_dhcp6_config_and_statistic_response_that_is_valid_should_return_every_metric( + valid_dhcp6, responsequeue +): config, statistics, expected_metrics = valid_dhcp6 responsequeue.prefill("dhcp6", config, statistics) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=6, tzinfo=timezone.utc) assert set(source.fetch_metrics()) == set(expected_metrics) -def test_dhcp4_config_and_statistic_response_that_is_valid_should_return_every_metric(valid_dhcp4, responsequeue): + +def test_dhcp4_config_and_statistic_response_that_is_valid_should_return_every_metric( + valid_dhcp4, responsequeue +): config, statistics, expected_metrics = valid_dhcp4 responsequeue.prefill("dhcp4", config, statistics) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4, tzinfo=timezone.utc) assert set(source.fetch_metrics()) == set(expected_metrics) -@pytest.mark.parametrize("status", [status for status in KeaStatus if status != KeaStatus.SUCCESS]) -def test_config_response_with_error_status_should_raise_KeaError(valid_dhcp4, responsequeue, status): + +@pytest.mark.parametrize( + "status", [status for status in KeaStatus if status != KeaStatus.SUCCESS] +) +def test_config_response_with_error_status_should_raise_KeaError( + valid_dhcp4, responsequeue, status +): """ If Kea responds with an error while fetching the Kea DHCP's config during fetch_metrics(), we cannot continue further, so we fail. @@ -35,7 +46,10 @@ def test_config_response_with_error_status_should_raise_KeaError(valid_dhcp4, re with pytest.raises(KeaError): source.fetch_metrics() -def test_any_response_with_invalid_format_should_raise_KeaError(valid_dhcp4, responsequeue): + +def test_any_response_with_invalid_format_should_raise_KeaError( + valid_dhcp4, responsequeue +): """ If Kea responds with an invalid format (i.e. in an unrecognizable way), we should fail loudly, because chances are either the host we're sending @@ -74,7 +88,9 @@ def test_any_response_with_invalid_format_should_raise_KeaError(valid_dhcp4, res source.fetch_metrics() -def test_all_responses_is_empty_but_valid_should_yield_no_metrics(valid_dhcp4, responsequeue): +def test_all_responses_is_empty_but_valid_should_yield_no_metrics( + valid_dhcp4, responsequeue +): """ If the Kea DHCP server we query does not have any subnets configured, the correct thing to do is to return an empty iterable, (as opposed to failing). @@ -91,7 +107,9 @@ def test_all_responses_is_empty_but_valid_should_yield_no_metrics(valid_dhcp4, r responsequeue.clear() responsequeue.prefill("dhcp4", config, None) - responsequeue.add("statistic-get", lambda arguments, **_: kearesponse({arguments["name"]: []})) + responsequeue.add( + "statistic-get", lambda arguments, **_: kearesponse({arguments["name"]: []}) + ) assert list(source.fetch_metrics()) == [] responsequeue.clear() @@ -109,215 +127,226 @@ def valid_dhcp6(): "renew-timer": 1000, "rebind-timer": 2000, "preferred-lifetime": 3000, - - "interfaces-config": { - "interfaces": [ "eth0" ] - }, - - "lease-database": { - "type": "memfile", - "persist": True, - "name": "/var/lib/kea/dhcp6.leases" - }, - - "subnet6": [ - { - "id": 1, - "subnet": "2001:db8:1:1::/64", - "pools": [ - { - "pool": "2001:db8:1:1::1-2001:db8:1:1::ffff" - } - ] - }, - { - "id": 2, - "subnet": "2001:db8:1:2::/64", - "pools": [ + "interfaces-config": {"interfaces": ["eth0"]}, + "lease-database": { + "type": "memfile", + "persist": True, + "name": "/var/lib/kea/dhcp6.leases", + }, + "subnet6": [ { - "pool": "2001:db8:1:2::1-2001:db8:1:2::ffff" + "id": 1, + "subnet": "2001:db8:1:1::/64", + "pools": [{"pool": "2001:db8:1:1::1-2001:db8:1:1::ffff"}], }, { - "pool": "2001:db8:1:2::1:0/112" - } - ] - } - ], - "shared-networks": [ - { - "name": "shared-network-1", - "subnet6": [ - { - "id": 3, - "subnet": "2001:db8:1:3::/64", + "id": 2, + "subnet": "2001:db8:1:2::/64", + "pools": [ + {"pool": "2001:db8:1:2::1-2001:db8:1:2::ffff"}, + {"pool": "2001:db8:1:2::1:0/112"}, + ], }, + ], + "shared-networks": [ { - "id": 4, - "subnet": "2001:db8:1:4::/64", - } - ] - }, - { - "name": "shared-network-2", - "subnet6": [ + "name": "shared-network-1", + "subnet6": [ + { + "id": 3, + "subnet": "2001:db8:1:3::/64", + }, + { + "id": 4, + "subnet": "2001:db8:1:4::/64", + }, + ], + }, { - "id": 5, - "subnet": "2001:db8:1:5::/64", - } - ] + "name": "shared-network-2", + "subnet6": [ + { + "id": 5, + "subnet": "2001:db8:1:5::/64", + } + ], + }, + ], } - ], - } } statistics = { "subnet[1].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140438" - ], - [ - 0, - "2024-07-05 20:44:54.230608" - ], - [ - 1, - "2024-07-05 09:15:05.626594" - ], - ], - "subnet[1].declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401071" - ] - ], - "subnet[1].total-addresses": [ - [ - 239, - "2024-07-03 16:13:59.401058" - ] + [1, "2024-07-22 09:06:58.140438"], + [0, "2024-07-05 20:44:54.230608"], + [1, "2024-07-05 09:15:05.626594"], ], + "subnet[1].declined-addresses": [[0, "2024-07-03 16:13:59.401071"]], + "subnet[1].total-addresses": [[239, "2024-07-03 16:13:59.401058"]], "subnet[2].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140439" - ], - [ - 1, - "2024-07-05 20:44:54.230609" - ], - [ - 2, - "2024-07-05 09:15:05.626595" - ], - ], - "subnet[2].declined-addresses": [ - [ - 1, - "2024-07-03 16:13:59.401072" - ] - ], - "subnet[2].total-addresses": [ - [ - 240, - "2024-07-03 16:13:59.401059" - ] + [1, "2024-07-22 09:06:58.140439"], + [1, "2024-07-05 20:44:54.230609"], + [2, "2024-07-05 09:15:05.626595"], ], + "subnet[2].declined-addresses": [[1, "2024-07-03 16:13:59.401072"]], + "subnet[2].total-addresses": [[240, "2024-07-03 16:13:59.401059"]], "subnet[3].assigned-addresses": [ - [ - 4, - "2024-07-22 09:06:58.140439" - ], - [ - 5, - "2024-07-05 20:44:54.230609" - ], - ], - "subnet[3].declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401072" - ] - ], - "subnet[3].total-addresses": [ - [ - 241, - "2024-07-03 16:13:59.401059" - ] + [4, "2024-07-22 09:06:58.140439"], + [5, "2024-07-05 20:44:54.230609"], ], + "subnet[3].declined-addresses": [[0, "2024-07-03 16:13:59.401072"]], + "subnet[3].total-addresses": [[241, "2024-07-03 16:13:59.401059"]], "subnet[4].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140439" - ], - [ - 1, - "2024-07-05 20:44:54.230609" - ], - ], - "subnet[4].declined-addresses": [ - [ - 1, - "2024-07-03 16:13:59.401072" - ] - ], - "subnet[4].total-addresses": [ - [ - 242, - "2024-07-03 16:13:59.401059" - ] + [1, "2024-07-22 09:06:58.140439"], + [1, "2024-07-05 20:44:54.230609"], ], + "subnet[4].declined-addresses": [[1, "2024-07-03 16:13:59.401072"]], + "subnet[4].total-addresses": [[242, "2024-07-03 16:13:59.401059"]], "subnet[5].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140439" - ], - [ - 1, - "2024-07-05 20:44:54.230609" - ], - ], - "subnet[5].declined-addresses": [ - [ - 1, - "2024-07-03 16:13:59.401072" - ] + [1, "2024-07-22 09:06:58.140439"], + [1, "2024-07-05 20:44:54.230609"], ], - "subnet[5].total-addresses": [ - [ - 243, - "2024-07-03 16:13:59.401059" - ] - ], - + "subnet[5].declined-addresses": [[1, "2024-07-03 16:13:59.401072"]], + "subnet[5].total-addresses": [[243, "2024-07-03 16:13:59.401059"]], } expected_metrics = [ - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.CUR, 0), - DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.TOUCH, 0), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), IP("2001:db8:1:1::/64"), DhcpMetricKey.MAX, 239), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.CUR, 2), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.TOUCH, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:2::/64"), DhcpMetricKey.MAX, 240), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.CUR, 4), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.CUR, 5), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.TOUCH, 0), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:3::/64"), DhcpMetricKey.MAX, 241), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.TOUCH, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:4::/64"), DhcpMetricKey.MAX, 242), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.TOUCH, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:5::/64"), DhcpMetricKey.MAX, 243), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), + IP("2001:db8:1:1::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), + IP("2001:db8:1:1::/64"), + DhcpMetricKey.CUR, + 0, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), + IP("2001:db8:1:1::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), + IP("2001:db8:1:1::/64"), + DhcpMetricKey.TOUCH, + 0, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), + IP("2001:db8:1:1::/64"), + DhcpMetricKey.MAX, + 239, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("2001:db8:1:2::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("2001:db8:1:2::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), + IP("2001:db8:1:2::/64"), + DhcpMetricKey.CUR, + 2, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("2001:db8:1:2::/64"), + DhcpMetricKey.TOUCH, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("2001:db8:1:2::/64"), + DhcpMetricKey.MAX, + 240, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("2001:db8:1:3::/64"), + DhcpMetricKey.CUR, + 4, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("2001:db8:1:3::/64"), + DhcpMetricKey.CUR, + 5, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("2001:db8:1:3::/64"), + DhcpMetricKey.TOUCH, + 0, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("2001:db8:1:3::/64"), + DhcpMetricKey.MAX, + 241, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("2001:db8:1:4::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("2001:db8:1:4::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("2001:db8:1:4::/64"), + DhcpMetricKey.TOUCH, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("2001:db8:1:4::/64"), + DhcpMetricKey.MAX, + 242, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("2001:db8:1:5::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("2001:db8:1:5::/64"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("2001:db8:1:5::/64"), + DhcpMetricKey.TOUCH, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("2001:db8:1:5::/64"), + DhcpMetricKey.MAX, + 243, + ), ] return config, statistics, expected_metrics + @pytest.fixture def valid_dhcp4(): config = { @@ -326,215 +355,226 @@ def valid_dhcp4(): "renew-timer": 1000, "rebind-timer": 2000, "preferred-lifetime": 3000, - - "interfaces-config": { - "interfaces": [ "eth0" ] - }, - - "lease-database": { - "type": "memfile", - "persist": True, - "name": "/var/lib/kea/dhcp6.leases" - }, - - "subnet4": [ - { - "id": 1, - "subnet": "192.0.1.0/24", - "pools": [ - { - "pool": "192.0.1.1-192.0.1.10" - } - ] - }, - { - "id": 2, - "subnet": "192.0.2.0/24", - "pools": [ + "interfaces-config": {"interfaces": ["eth0"]}, + "lease-database": { + "type": "memfile", + "persist": True, + "name": "/var/lib/kea/dhcp6.leases", + }, + "subnet4": [ { - "pool": "192.0.2.1-192.0.2.10" + "id": 1, + "subnet": "192.0.1.0/24", + "pools": [{"pool": "192.0.1.1-192.0.1.10"}], }, { - "pool": "192.0.2.128/25" - } - ] - } - ], - "shared-networks": [ - { - "name": "shared-network-1", - "subnet4": [ - { - "id": 3, - "subnet": "192.0.3.0/24", + "id": 2, + "subnet": "192.0.2.0/24", + "pools": [ + {"pool": "192.0.2.1-192.0.2.10"}, + {"pool": "192.0.2.128/25"}, + ], }, + ], + "shared-networks": [ { - "id": 4, - "subnet": "192.0.4.0/24", - } - ] - }, - { - "name": "shared-network-2", - "subnet4": [ + "name": "shared-network-1", + "subnet4": [ + { + "id": 3, + "subnet": "192.0.3.0/24", + }, + { + "id": 4, + "subnet": "192.0.4.0/24", + }, + ], + }, { - "id": 5, - "subnet": "192.0.5.0/24", - } - ] + "name": "shared-network-2", + "subnet4": [ + { + "id": 5, + "subnet": "192.0.5.0/24", + } + ], + }, + ], } - ], - } } statistics = { "subnet[1].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140438" - ], - [ - 0, - "2024-07-05 20:44:54.230608" - ], - [ - 1, - "2024-07-05 09:15:05.626594" - ], - ], - "subnet[1].declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401071" - ] - ], - "subnet[1].total-addresses": [ - [ - 239, - "2024-07-03 16:13:59.401058" - ] + [1, "2024-07-22 09:06:58.140438"], + [0, "2024-07-05 20:44:54.230608"], + [1, "2024-07-05 09:15:05.626594"], ], + "subnet[1].declined-addresses": [[0, "2024-07-03 16:13:59.401071"]], + "subnet[1].total-addresses": [[239, "2024-07-03 16:13:59.401058"]], "subnet[2].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140439" - ], - [ - 1, - "2024-07-05 20:44:54.230609" - ], - [ - 2, - "2024-07-05 09:15:05.626595" - ], - ], - "subnet[2].declined-addresses": [ - [ - 1, - "2024-07-03 16:13:59.401072" - ] - ], - "subnet[2].total-addresses": [ - [ - 240, - "2024-07-03 16:13:59.401059" - ] + [1, "2024-07-22 09:06:58.140439"], + [1, "2024-07-05 20:44:54.230609"], + [2, "2024-07-05 09:15:05.626595"], ], + "subnet[2].declined-addresses": [[1, "2024-07-03 16:13:59.401072"]], + "subnet[2].total-addresses": [[240, "2024-07-03 16:13:59.401059"]], "subnet[3].assigned-addresses": [ - [ - 4, - "2024-07-22 09:06:58.140439" - ], - [ - 5, - "2024-07-05 20:44:54.230609" - ], - ], - "subnet[3].declined-addresses": [ - [ - 0, - "2024-07-03 16:13:59.401072" - ] - ], - "subnet[3].total-addresses": [ - [ - 241, - "2024-07-03 16:13:59.401059" - ] + [4, "2024-07-22 09:06:58.140439"], + [5, "2024-07-05 20:44:54.230609"], ], + "subnet[3].declined-addresses": [[0, "2024-07-03 16:13:59.401072"]], + "subnet[3].total-addresses": [[241, "2024-07-03 16:13:59.401059"]], "subnet[4].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140439" - ], - [ - 1, - "2024-07-05 20:44:54.230609" - ], - ], - "subnet[4].declined-addresses": [ - [ - 1, - "2024-07-03 16:13:59.401072" - ] - ], - "subnet[4].total-addresses": [ - [ - 242, - "2024-07-03 16:13:59.401059" - ] + [1, "2024-07-22 09:06:58.140439"], + [1, "2024-07-05 20:44:54.230609"], ], + "subnet[4].declined-addresses": [[1, "2024-07-03 16:13:59.401072"]], + "subnet[4].total-addresses": [[242, "2024-07-03 16:13:59.401059"]], "subnet[5].assigned-addresses": [ - [ - 1, - "2024-07-22 09:06:58.140439" - ], - [ - 1, - "2024-07-05 20:44:54.230609" - ], - ], - "subnet[5].declined-addresses": [ - [ - 1, - "2024-07-03 16:13:59.401072" - ] + [1, "2024-07-22 09:06:58.140439"], + [1, "2024-07-05 20:44:54.230609"], ], - "subnet[5].total-addresses": [ - [ - 243, - "2024-07-03 16:13:59.401059" - ] - ], - + "subnet[5].declined-addresses": [[1, "2024-07-03 16:13:59.401072"]], + "subnet[5].total-addresses": [[243, "2024-07-03 16:13:59.401059"]], } expected_metrics = [ - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.CUR, 0), - DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.TOUCH, 0), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), IP("192.0.1.0/24"), DhcpMetricKey.MAX, 239), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.CUR, 2), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.TOUCH, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.2.0/24"), DhcpMetricKey.MAX, 240), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.CUR, 4), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.CUR, 5), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.TOUCH, 0), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.3.0/24"), DhcpMetricKey.MAX, 241), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.TOUCH, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.4.0/24"), DhcpMetricKey.MAX, 242), - DhcpMetric(datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.CUR, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.TOUCH, 1), - DhcpMetric(datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.5.0/24"), DhcpMetricKey.MAX, 243), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), + IP("192.0.1.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), + IP("192.0.1.0/24"), + DhcpMetricKey.CUR, + 0, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), + IP("192.0.1.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), + IP("192.0.1.0/24"), + DhcpMetricKey.TOUCH, + 0, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), + IP("192.0.1.0/24"), + DhcpMetricKey.MAX, + 239, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("192.0.2.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("192.0.2.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), + IP("192.0.2.0/24"), + DhcpMetricKey.CUR, + 2, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("192.0.2.0/24"), + DhcpMetricKey.TOUCH, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("192.0.2.0/24"), + DhcpMetricKey.MAX, + 240, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("192.0.3.0/24"), + DhcpMetricKey.CUR, + 4, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("192.0.3.0/24"), + DhcpMetricKey.CUR, + 5, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("192.0.3.0/24"), + DhcpMetricKey.TOUCH, + 0, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("192.0.3.0/24"), + DhcpMetricKey.MAX, + 241, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("192.0.4.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("192.0.4.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("192.0.4.0/24"), + DhcpMetricKey.TOUCH, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("192.0.4.0/24"), + DhcpMetricKey.MAX, + 242, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + IP("192.0.5.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + IP("192.0.5.0/24"), + DhcpMetricKey.CUR, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), + IP("192.0.5.0/24"), + DhcpMetricKey.TOUCH, + 1, + ), + DhcpMetric( + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + IP("192.0.5.0/24"), + DhcpMetricKey.MAX, + 243, + ), ] return config, statistics, expected_metrics + def kearesponse(val, status=KeaStatus.SUCCESS): return f''' [ @@ -595,7 +635,9 @@ def new_post_function(url, *args, data="{}", **kwargs): if fifo: first = fifo[0] if callable(first): - text = first(arguments=data.get("arguments", {}), service=data.get("service", [])) + text = first( + arguments=data.get("arguments", {}), service=data.get("service", []) + ) else: text = str(first) fifo.popleft() @@ -628,10 +670,15 @@ def clear_command_responses(): def prefill_command_responses(expected_service, config=None, statistics=None): def config_get_response(arguments, service): - assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + assert service == [ + expected_service + ], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" return kearesponse(config) + def statistic_get_response(arguments, service): - assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + assert service == [ + expected_service + ], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" return kearesponse({arguments["name"]: statistics[arguments["name"]]}) if config is not None: From 6a4b5eb0f4b33375ec51e47b6c3654021d2bc089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 24 Jul 2024 11:16:12 +0200 Subject: [PATCH 56/77] Fix imports --- python/nav/dhcp/generic_metrics.py | 3 ++- python/nav/dhcp/kea_metrics.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 1848c013b4..3aa77d272b 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -4,6 +4,7 @@ from nav.metrics import carbon from nav.metrics.names import escape_metric_name from typing import Iterator +from datetime import datetime class DhcpMetricKey(Enum): @@ -17,7 +18,7 @@ def __str__(self): @dataclass(frozen=True) class DhcpMetric: - timestamp: int + timestamp: datetime subnet_prefix: IP key: DhcpMetricKey value: int diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 2b5e5e413d..40cac63411 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -8,8 +8,6 @@ import logging from requests import RequestException, JSONDecodeError import requests -import calendar -import time import json from enum import IntEnum From 3582bd330726c16d854fb79c74d6932a50529816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 24 Jul 2024 19:24:55 +0200 Subject: [PATCH 57/77] Return List type, not Iterator in kea_metrics.fetch_metrics --- python/nav/dhcp/kea_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 40cac63411..0bb934bd6b 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -1,5 +1,5 @@ from IPy import IP -from typing import Iterator, Optional +from typing import Optional from itertools import chain from nav.dhcp.generic_metrics import DhcpMetricSource from nav.errors import GeneralException @@ -49,7 +49,7 @@ def __init__( self.timeout = timeout self.tzinfo = tzinfo - def fetch_metrics(self) -> Iterator[DhcpMetric]: + def fetch_metrics(self) -> list[DhcpMetric]: """ Fetch total addresses, assigned addresses, and declined addresses of all subnets the Kea DHCP server serving ip version `dhcp_version` maintains. From 5826f2208bcba3b325ad128e223a9fdcfbdcc427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 24 Jul 2024 20:10:49 +0200 Subject: [PATCH 58/77] Enhance docstrings --- python/nav/dhcp/kea_metrics.py | 82 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 0bb934bd6b..1cd8e30b68 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -16,9 +16,9 @@ class KeaDhcpMetricSource(DhcpMetricSource): """ - Sends http requests to Kea Control Agent on `self.rest_uri` to fetch metrics - from all subnets managed by the Kea DHCP server (serving ip version - `dhcp_version` addresses) that the Kea Control Agent controls. + Communicates with a Kea Control Agent and fetches metrics from all + subnets managed by the Kea DHCP server serving a specific ip + version that is controlled the Kea Control Agent. """ def __init__( @@ -33,13 +33,14 @@ def __init__( **kwargs, ): """ - Instantiate a KeaDhcpMetricSource that fetches information via the Kea - Control Agent listening to `port` on `address`. + Returns a KeaDhcpMetricSource that fetches DHCP metrics via + the Kea Control Agent listening to `port` on `address`. - :param https: if True, use https. Otherwise, use http + :param https: if True, use https. Otherwise, use http :param dhcp_version: ip version served by Kea DHCP server - :param timeout: how long to wait for http response from Kea Control Agent before timing out - :param tzinfo: the timezone of the Kea Control Agent. + :param timeout: how long to wait for a http response from + the Kea Control Agent before timing out + :param tzinfo: the timezone of the Kea Control Agent. """ super(*args, **kwargs) scheme = "https" if https else "http" @@ -51,15 +52,27 @@ def __init__( def fetch_metrics(self) -> list[DhcpMetric]: """ - Fetch total addresses, assigned addresses, and declined addresses of all - subnets the Kea DHCP server serving ip version `dhcp_version` maintains. + Returns a list of DHCP metrics. For each subnet and + DhcpMetric-key combination, there is at least one + corresponding metric in the returned list if no errors occur. + + If the Kea Control Agent responds with an empty response to + one or more of the requests for some metric(s), these metrics + will be missing in the returned list, but a list is still + succesfully returned. Other errors while requesting metrics + will cause a fitting subclass of KeaException to be raised: + + Communication errors (HTTP errors, JSON errors, access control + errors) causes a KeaException that is reraised from the + specific communication error to be raised. + + If the Kea Control Agent doesn't support the 'config-get' and + 'statistic-get' commands, then a KeaUnsupported exception is + raised. + + General errors reported by the Kea Control Agent causes a + KeaError to be raised. """ - - metric_keys = ( - ("total-addresses", DhcpMetricKey.MAX), - ("assigned-addresses", DhcpMetricKey.CUR), - ("declined-addresses", DhcpMetricKey.TOUCH), - ) metrics = [] with requests.Session() as s: @@ -94,8 +107,8 @@ def fetch_metrics(self) -> list[DhcpMetric]: def _fetch_config(self, session: requests.Session) -> dict: """ - Fetch the current config of the Kea DHCP server serving ip version - `dhcp_version` + Returns the current config of the Kea DHCP server that the Kea + Control Agent controls """ if ( self.dhcp_config is None @@ -115,8 +128,8 @@ def _fetch_config(self, session: requests.Session) -> dict: def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: """ - Fetch the hash of the current config of the Kea DHCP server serving ip - version `dhcp_version` + Returns the hash of the current config of the Kea DHCP server + that the Kea Control Agent controls """ return ( self._send_query(session, "config-hash-get") @@ -126,13 +139,18 @@ def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict: """ - Send `command` to the Kea Control Agent. An exception is raised iff - there was an HTTP related error while sending `command` or a response - does not look like it is coming from a Kea Control Agent or if the - request likely was rejected by the Kea Control Agent. All raised - exceptions are of type `KeaError`. Occurrences of valid Kea error - responses such as those with result == KeaStatus.ERROR are logged as an - error, and result in an empty dictionary being returned. + Returns the response from the Kea Control Agent to the query + with command `command`. Additional keyword arguments to this + function will be passed as arguments to the command. + + Communication errors (HTTP errors, JSON errors, access control + errors, unrecognized json response formats) causes a + KeaException to be raised. If possible, it is reraised from a + more descriptive error such as a HTTPError. + + Valid Kea Control Agent responses that indicate a failure on + the server-end causes a descriptive subclass of KeaException + to be raised. """ postdata = json.dumps( { @@ -197,8 +215,8 @@ def _parsetime(self, timestamp: str) -> int: def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: """ - List the id and prefix of subnets listed in the Kea DHCP configuration - `config` + List the id and prefix of subnets listed in the Kea DHCP + configuration `config` """ subnets = [] subnetkey = f"subnet{ip_version}" @@ -222,14 +240,8 @@ class KeaError(GeneralException): class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" - - # Successful operation. SUCCESS = 0 - # General failure. ERROR = 1 - # Command is not supported. UNSUPPORTED = 2 - # Successful operation, but failed to produce any results. EMPTY = 3 - # Unsuccessful operation due to a conflict between the command arguments and the server state. CONFLICT = 4 From 30297e05a28302c214720def574b73a644a8f7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Wed, 24 Jul 2024 20:11:53 +0200 Subject: [PATCH 59/77] Alter exception handling --- python/nav/dhcp/kea_metrics.py | 154 ++++++++++++++++------- tests/unittests/dhcp/kea_metrics_test.py | 10 +- 2 files changed, 112 insertions(+), 52 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 1cd8e30b68..1022d634b1 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -116,14 +116,13 @@ def _fetch_config(self, session: requests.Session) -> dict: or self._fetch_config_hash(session) != dhcp_confighash ): response = self._send_query(session, "config-get") - status = response.get("result", KeaStatus.ERROR) - arguments = response.get("arguments", {}) - self.dhcp_config = arguments.get(f"Dhcp{self.dhcp_version}", None) - if self.dhcp_config is None or status != KeaStatus.SUCCESS: - raise KeaError( - "Could not fetch configuration of Kea DHCP server from Kea Control " - f"Agent at {self.rest_uri}" - ) + try: + self.dhcp_config = response["arguments"][f"Dhcp{self.dhcp_version}"] + except KeyError as err: + raise KeaException( + "Unrecognizable response to the 'config-get' request", + {"Response": response} + ) from err return self.dhcp_config def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: @@ -131,11 +130,16 @@ def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: Returns the hash of the current config of the Kea DHCP server that the Kea Control Agent controls """ - return ( - self._send_query(session, "config-hash-get") - .get("arguments", {}) - .get("hash", None) - ) + try: + return ( + self._send_query(session, "config-hash-get") + .get("arguments", {}) + .get("hash", None) + ) + except KeaUnsupported as err: + _logger.debug(str(err)) + return None + def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict: """ @@ -152,60 +156,93 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict the server-end causes a descriptive subclass of KeaException to be raised. """ - postdata = json.dumps( + post_data = json.dumps( { "command": command, "arguments": {**kwargs}, "service": [f"dhcp{self.dhcp_version}"], } ) - _logger.info( - "send_query: Post request to Kea Control Agent at %s with data %s", - self.rest_uri, - postdata, - ) + request_summary = { + "Description": f"Sending request to Kea Control Agent at {self.rest_uri}", + "Status": "sending", + "Location": self.rest_uri, + "Request": post_data, + } + + _logger.debug(request_summary) + + request_summary["Validity"] = "Invalid Kea response" try: responses = session.post( self.rest_uri, - data=postdata, + data=post_data, timeout=self.timeout, ) + request_summary["Status"] = "complete" + request_summary["Response"] = responses.text + request_summary["HTTP Status"] = responses.status_code responses = responses.json() - except RequestException as err: - raise KeaError( - f"HTTP related error when requesting Kea Control Agent at {self.rest_uri}", - ) from err except JSONDecodeError as err: - raise KeaError( - f"Uri {self.rest_uri} most likely not pointing at a Kea " - f"Control Agent (expected json, responded with: {responses!r})", + raise KeaException( + "Server does not look like a Kea Control Agent; " + "expected response content to be JSON", + request_summary + ) from err + except RequestException as err: + raise KeaException( + "HTTP-related error during request to server", + request_summary ) from err if not isinstance(responses, list): # See https://kea.readthedocs.io/en/kea-2.6.0/arm/ctrl-channel.html#control-agent-command-response-format - raise KeaError( - f"Kea Control Agent at {self.rest_uri} have likely rejected " - f"a query (responded with: {responses!r})" + raise KeaException( + "Invalid response; server has likely rejected a query", + request_summary ) if not (len(responses) == 1 and "result" in responses[0]): - # "We've only sent the command to *one* service. Thus responses should contain *one* response." - raise KeaError( - f"Uri {self.rest_uri} most likely not pointing at a Kea " - "Control Agent (expected json list with one object having " - f"key 'result', responded with: {responses!r})", + # We've only sent the command to *one* service. Thus responses should contain *one* response. + raise KeaException( + "Server does not look like a Kea Control Agent; " + "expected response content to be a JSON list " + "of a single object that has 'result' as one of its keys. ", + request_summary ) + request_summary["Validity"] = "Valid Kea response" + + _logger.debug(request_summary) + response = responses[0] - if response["result"] == KeaStatus.SUCCESS: + status = response["result"] + + if status == KeaStatus.SUCCESS: return response - else: - _logger.error( - "send_query: Kea at %s did not succeed fulfilling query %s " - "(responded with: %r) ", - self.rest_uri, - postdata, - responses, + elif status == KeaStatus.UNSUPPORTED: + raise KeaUnsupported( + "Command '{command}' not supported by Kea", + request_summary + ) + elif status == KeaStatus.EMPTY: + raise KeaEmpty( + "Requested resource not found", + request_summary ) - return {} + elif status == KeaStatus.ERROR: + raise KeaError( + "Kea failed during command processing", + request_summary + ) + elif status == KeaStatus.CONFLICT: + raise KeaConflict( + "Kea failed apply requested changes", + request_summary + ) + raise KeaError( + "Kea returned an unkown status response", + request_summary + ) + def _parsetime(self, timestamp: str) -> int: """Parse the timestamp string used in Kea's timeseries into unix time""" @@ -227,16 +264,39 @@ def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: id = subnet.get("id", None) prefix = subnet.get("subnet", None) if id is None or prefix is None: - msg = "subnets: id or prefix missing from a subnet's configuration: %r" - _logger.warning(msg, subnet) + _logger.warning( + "id or prefix missing from a subnet's configuration: %r", + subnet + ) continue subnets.append((id, IP(prefix))) return subnets -class KeaError(GeneralException): +class KeaException(GeneralException): """Error related to interaction with a Kea Control Agent""" + def __init__(self, message: str = "", details: dict[str, str] = {}): + self.message = message + self.details = details + + def __str__(self) -> str: + doc = self.__doc__ + message = "" + details = "" + if self.message: + message = f": {self.message}" + if self.details: + details = "".join(f"\n{label}: {info}" for label, info in self.details) + return "".join(doc, message, details) +class KeaError(KeaException): + """General error or failure occurred during command processing on server""" +class KeaUnsupported(KeaException): + """Unsupported command""" +class KeaEmpty(KeaException): + """Completed command but requested resource not found""" +class KeaConflict(KeaException): + """The requested change conflicts with the server's state""" class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent.""" diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index c232b89d35..d451d38b09 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -43,7 +43,7 @@ def test_config_response_with_error_status_should_raise_KeaError( responsequeue.prefill("dhcp4", None, statistics) responsequeue.add("config-get", kearesponse(config, status=status)) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4) - with pytest.raises(KeaError): + with pytest.raises(KeaException): source.fetch_metrics() @@ -60,21 +60,21 @@ def test_any_response_with_invalid_format_should_raise_KeaError( source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4) responsequeue.add("config-get", "{}") - with pytest.raises(KeaError): + with pytest.raises(KeaException): source.fetch_metrics() responsequeue.clear() responsequeue.prefill("dhcp4", config, None) responsequeue.add("statistic-get", "{}") - with pytest.raises(KeaError): + with pytest.raises(KeaException): source.fetch_metrics() responsequeue.clear() responsequeue.prefill("dhcp4", None, statistics) responsequeue.add("config-get", "{}") - with pytest.raises(KeaError): + with pytest.raises(KeaException): source.fetch_metrics() responsequeue.clear() @@ -84,7 +84,7 @@ def test_any_response_with_invalid_format_should_raise_KeaError( config["Dhcp4"]["hash"] = "foo" responsequeue.prefill("dhcp4", config, statistics) responsequeue.add("config-hash-get", "{}") - with pytest.raises(KeaError): + with pytest.raises(KeaException): source.fetch_metrics() From a9509fb1e9d1cfaf4e8cb7b0e8afbab70979b7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 14:58:51 +0200 Subject: [PATCH 60/77] Rename keys in DhcpMetricKey and remove TOUCH key what: prior to this commit, the key for a specific metric (i.e. the name of that metric used by NAV) had the same naming convention as dhcpd-pools(1), e.g. "cur" was the name used for the "amount of addresses currently assigned to dhcp clients on this subnet" and "max" was the name used for the "total amount of addresses controlled by this subnet". DhcpMetricKey.CUR and DhcpMetricKey.MAX, however, is not very descriptive, so I changed the key names to be DhcpMetricKey.TOTAL and DhcpMetricKey.ASSIGNED. DhcpMetricKey.TOUCH was removed all together because it seems to me like this is not a common metric to be reported by dhcp servers (dhcpd-pools(1) uses "touch" to mean the number of assigned addresses that has timed out but that is not yet marked as re-assignable by the dhcp-server). --- python/nav/dhcp/generic_metrics.py | 8 +- python/nav/dhcp/kea_metrics.py | 20 +++- tests/unittests/dhcp/kea_metrics_test.py | 128 ++++++----------------- 3 files changed, 52 insertions(+), 104 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 3aa77d272b..cac8e58afb 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -8,12 +8,10 @@ class DhcpMetricKey(Enum): - MAX = "max" # total addresses - CUR = "cur" # assigned addresses - TOUCH = "touch" # touched addresses - + TOTAL = "total" # total addresses managed by dhcp + ASSIGNED = "assigned" # assigned addresses def __str__(self): - return self.name.lower() # For use in graphite path + return self.value # graphite key @dataclass(frozen=True) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 1022d634b1..ee3ca8f02f 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -73,15 +73,22 @@ def fetch_metrics(self) -> list[DhcpMetric]: General errors reported by the Kea Control Agent causes a KeaError to be raised. """ + metric_keys = ( + ("total-addresses", DhcpMetricKey.TOTAL), + ("assigned-addresses", DhcpMetricKey.ASSIGNED), + ) metrics = [] with requests.Session() as s: config = self._fetch_config(s) subnets = _subnets_of_config(config, self.dhcp_version) - for subnetid, netprefix in subnets: + for subnet_id, netprefix in subnets: for kea_key, nav_key in metric_keys: - kea_name = f"subnet[{subnetid}].{kea_key}" - response = self._send_query(s, "statistic-get", name=kea_name) + kea_name = f"subnet[{subnet_id}].{kea_key}" + try: + response = self._send_query(s, "statistic-get", name=kea_name) + except KeaEmpty: + continue timeseries = response.get("arguments", {}).get(kea_name, []) if len(timeseries) == 0: _logger.error( @@ -91,10 +98,13 @@ def fetch_metrics(self) -> list[DhcpMetric]: netprefix, kea_name, ) - for val, t in timeseries: - metric = DhcpMetric(self._parsetime(t), netprefix, nav_key, val) + for value, timestring in timeseries: + metric = DhcpMetric( + self._parsetime(timestring), netprefix, nav_key, value + ) metrics.append(metric) + newsubnets = _subnets_of_config(self._fetch_config(s), self.dhcp_version) if sorted(subnets) != sorted(newsubnets): _logger.error( diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index d451d38b09..92d1a7f70b 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -213,133 +213,103 @@ def valid_dhcp6(): DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), IP("2001:db8:1:1::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), IP("2001:db8:1:1::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 0, ), DhcpMetric( datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), IP("2001:db8:1:1::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), - IP("2001:db8:1:1::/64"), - DhcpMetricKey.TOUCH, - 0, - ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), IP("2001:db8:1:1::/64"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 239, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:2::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:2::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), IP("2001:db8:1:2::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 2, ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("2001:db8:1:2::/64"), - DhcpMetricKey.TOUCH, - 1, - ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:2::/64"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 240, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:3::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 4, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:3::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 5, ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("2001:db8:1:3::/64"), - DhcpMetricKey.TOUCH, - 0, - ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:3::/64"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 241, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:4::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:4::/64"), - DhcpMetricKey.CUR, - 1, - ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("2001:db8:1:4::/64"), - DhcpMetricKey.TOUCH, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:4::/64"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 242, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("2001:db8:1:5::/64"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("2001:db8:1:5::/64"), - DhcpMetricKey.CUR, - 1, - ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("2001:db8:1:5::/64"), - DhcpMetricKey.TOUCH, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("2001:db8:1:5::/64"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 243, ), ] @@ -441,133 +411,103 @@ def valid_dhcp4(): DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), IP("192.0.1.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), IP("192.0.1.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 0, ), DhcpMetric( datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), IP("192.0.1.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401071+00:00"), - IP("192.0.1.0/24"), - DhcpMetricKey.TOUCH, - 0, - ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), IP("192.0.1.0/24"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 239, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.2.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.2.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), IP("192.0.2.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 2, ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("192.0.2.0/24"), - DhcpMetricKey.TOUCH, - 1, - ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.2.0/24"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 240, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.3.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 4, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.3.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 5, ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("192.0.3.0/24"), - DhcpMetricKey.TOUCH, - 0, - ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.3.0/24"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 241, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.4.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.4.0/24"), - DhcpMetricKey.CUR, - 1, - ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("192.0.4.0/24"), - DhcpMetricKey.TOUCH, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.4.0/24"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 242, ), DhcpMetric( datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), IP("192.0.5.0/24"), - DhcpMetricKey.CUR, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), IP("192.0.5.0/24"), - DhcpMetricKey.CUR, - 1, - ), - DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401072+00:00"), - IP("192.0.5.0/24"), - DhcpMetricKey.TOUCH, + DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), IP("192.0.5.0/24"), - DhcpMetricKey.MAX, + DhcpMetricKey.TOTAL, 243, ), ] From 17b5a81cef23f851a535a011db6d65c389070736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:05:53 +0200 Subject: [PATCH 61/77] Use NAVs configured carbon host and port by default when sending metrics --- python/nav/dhcp/generic_metrics.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index cac8e58afb..60f52ec9f4 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from enum import Enum from IPy import IP -from nav.metrics import carbon +from nav.metrics import carbon, CONFIG from nav.metrics.names import escape_metric_name from typing import Iterator from datetime import datetime @@ -41,7 +41,11 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: """ raise NotImplementedError - def fetch_metrics_to_graphite(self, host, port): + def fetch_metrics_to_graphite( + self, + host=CONFIG.get("carbon", "host"), + port=CONFIG.getint("carbon", "port") + ): graphite_metrics = [] for metric in self.fetch_metrics(): graphite_path = f"{self.graphite_prefix}.{escape_metric_name(metric.subnet_prefix.strNormal())}.{metric.key}" From 1b0d67082eb76ea168844c71a37077fc2e7ebf5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:14:11 +0200 Subject: [PATCH 62/77] Update docstrings what: Modify the docstrings so that they all follow the same pattern; i.e. they all begin by describing what they return. --- python/nav/dhcp/generic_metrics.py | 11 +++++++++-- python/nav/dhcp/kea_metrics.py | 30 ++++++++++++++++-------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 60f52ec9f4..c5d32c0f8b 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -36,8 +36,8 @@ def __init__(self, graphite_prefix="nav.dhcp"): def fetch_metrics(self) -> Iterator[DhcpMetric]: """ - Fetch DhcpMetrics having keys `MAX`, `CUR`, `TOUCH` and `FREE` - for each subnet of the DHCP server at current point of time. + Fetch DhcpMetrics having keys `TOTAL` and `ASSIGNED` for each subnet of the + DHCP server at current point of time. """ raise NotImplementedError @@ -46,6 +46,13 @@ def fetch_metrics_to_graphite( host=CONFIG.get("carbon", "host"), port=CONFIG.getint("carbon", "port") ): + """ + Fetch metrics describing total amount of addresses + (DhcpMetricKey.TOTAL) and amount of addresses that have been + assigned to a client (DhcpMetricKey.ASSIGNED) for each subnet + of the DHCP server at current point of time and send the + metrics to the graphite server at `host` on `port`. + """ graphite_metrics = [] for metric in self.fetch_metrics(): graphite_path = f"{self.graphite_prefix}.{escape_metric_name(metric.subnet_prefix.strNormal())}.{metric.key}" diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index ee3ca8f02f..7066939e25 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -13,12 +13,12 @@ _logger = logging.getLogger(__name__) - class KeaDhcpMetricSource(DhcpMetricSource): """ - Communicates with a Kea Control Agent and fetches metrics from all - subnets managed by the Kea DHCP server serving a specific ip - version that is controlled the Kea Control Agent. + Communicates with a Kea Control Agent and fetches metrics for each + subnet managed by the Kea DHCP server serving a specific ip + version that is controlled by the Kea Control Agent (see + `KeaDhcpMetricSource.fetch_metrics`). """ def __init__( @@ -52,8 +52,9 @@ def __init__( def fetch_metrics(self) -> list[DhcpMetric]: """ - Returns a list of DHCP metrics. For each subnet and - DhcpMetric-key combination, there is at least one + Returns a list containing the most recent DHCP metrics for + each subnet managed by the Kea DHCP server. For each subnet + and DhcpMetric-key combination, there is at least one corresponding metric in the returned list if no errors occur. If the Kea Control Agent responds with an empty response to @@ -115,10 +116,11 @@ def fetch_metrics(self) -> list[DhcpMetric]: return metrics + def _fetch_config(self, session: requests.Session) -> dict: """ Returns the current config of the Kea DHCP server that the Kea - Control Agent controls + Control Agent controls. """ if ( self.dhcp_config is None @@ -138,7 +140,7 @@ def _fetch_config(self, session: requests.Session) -> dict: def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: """ Returns the hash of the current config of the Kea DHCP server - that the Kea Control Agent controls + that the Kea Control Agent controls. """ try: return ( @@ -262,8 +264,8 @@ def _parsetime(self, timestamp: str) -> int: def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: """ - List the id and prefix of subnets listed in the Kea DHCP - configuration `config` + Returns a list containing one (subnet-id, subnet-prefix) tuple per + subnet listed in the Kea DHCP configuration `config`. """ subnets = [] subnetkey = f"subnet{ip_version}" @@ -300,16 +302,16 @@ def __str__(self) -> str: return "".join(doc, message, details) class KeaError(KeaException): - """General error or failure occurred during command processing on server""" + """Kea failed during command processing""" class KeaUnsupported(KeaException): """Unsupported command""" class KeaEmpty(KeaException): - """Completed command but requested resource not found""" + """Requested resource not found""" class KeaConflict(KeaException): - """The requested change conflicts with the server's state""" + """Kea failed to apply requested changes due to conflicts with its server state""" class KeaStatus(IntEnum): - """Status of a response sent from a Kea Control Agent.""" + """Status of a response sent from a Kea Control Agent""" SUCCESS = 0 ERROR = 1 UNSUPPORTED = 2 From 2a5191e3efbde95226de7058089282769c340644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:21:26 +0200 Subject: [PATCH 63/77] Do not log the contents of a http request / response why: In the future, one might want to include sensitive information, such as passwords or tokens in requests, and a response from Kea might contain secrets, especially with regards to "config-get" responses, where a config might contain passwords. --- python/nav/dhcp/kea_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 7066939e25..d22f5c2fa7 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -179,7 +179,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict "Description": f"Sending request to Kea Control Agent at {self.rest_uri}", "Status": "sending", "Location": self.rest_uri, - "Request": post_data, + "Command": command, } _logger.debug(request_summary) @@ -192,7 +192,6 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict timeout=self.timeout, ) request_summary["Status"] = "complete" - request_summary["Response"] = responses.text request_summary["HTTP Status"] = responses.status_code responses = responses.json() except JSONDecodeError as err: From bea8d672f9185ca23f164aeaf69f88990beae6e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:23:58 +0200 Subject: [PATCH 64/77] Actually raise a KeaException on HTTP response error statuses --- python/nav/dhcp/kea_metrics.py | 52 +++++++++++++--------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index d22f5c2fa7..15338f4c35 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -156,34 +156,34 @@ def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict: """ Returns the response from the Kea Control Agent to the query - with command `command`. Additional keyword arguments to this - function will be passed as arguments to the command. + with command `command` instructed towards the Kea DHCP server. + Additional keyword arguments to this function will be passed + as arguments to the command. Communication errors (HTTP errors, JSON errors, access control errors, unrecognized json response formats) causes a KeaException to be raised. If possible, it is reraised from a - more descriptive error such as a HTTPError. + more descriptive error such as an HTTPError. Valid Kea Control Agent responses that indicate a failure on the server-end causes a descriptive subclass of KeaException to be raised. """ - post_data = json.dumps( - { - "command": command, - "arguments": {**kwargs}, - "service": [f"dhcp{self.dhcp_version}"], - } - ) request_summary = { "Description": f"Sending request to Kea Control Agent at {self.rest_uri}", "Status": "sending", "Location": self.rest_uri, "Command": command, } - _logger.debug(request_summary) + post_data = json.dumps( + { + "command": command, + "arguments": {**kwargs}, + "service": [f"dhcp{self.dhcp_version}"], + } + ) request_summary["Validity"] = "Invalid Kea response" try: responses = session.post( @@ -193,6 +193,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict ) request_summary["Status"] = "complete" request_summary["HTTP Status"] = responses.status_code + responses.raise_for_status() responses = responses.json() except JSONDecodeError as err: raise KeaException( @@ -213,7 +214,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict request_summary ) if not (len(responses) == 1 and "result" in responses[0]): - # We've only sent the command to *one* service. Thus responses should contain *one* response. + # `post-data` queries *one* service. Thus `responses` should contain *one* response. raise KeaException( "Server does not look like a Kea Control Agent; " "expected response content to be a JSON list " @@ -230,30 +231,15 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict if status == KeaStatus.SUCCESS: return response elif status == KeaStatus.UNSUPPORTED: - raise KeaUnsupported( - "Command '{command}' not supported by Kea", - request_summary - ) + raise KeaUnsupported(details=request_summary) elif status == KeaStatus.EMPTY: - raise KeaEmpty( - "Requested resource not found", - request_summary - ) + raise KeaEmpty(details=request_summary) elif status == KeaStatus.ERROR: - raise KeaError( - "Kea failed during command processing", - request_summary - ) + raise KeaError(details=request_summary) elif status == KeaStatus.CONFLICT: - raise KeaConflict( - "Kea failed apply requested changes", - request_summary - ) - raise KeaError( - "Kea returned an unkown status response", - request_summary - ) - + raise KeaConflict(details=request_summary) + raise KeaException("Kea returned an unkown status response", request_summary) + def _parsetime(self, timestamp: str) -> int: """Parse the timestamp string used in Kea's timeseries into unix time""" From 963e326e3f88530628dfd9a0befe7059a5be17c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:25:10 +0200 Subject: [PATCH 65/77] Fix wrong usage of str.join() function --- python/nav/dhcp/kea_metrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 15338f4c35..878dc28923 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -283,8 +283,9 @@ def __str__(self) -> str: if self.message: message = f": {self.message}" if self.details: - details = "".join(f"\n{label}: {info}" for label, info in self.details) - return "".join(doc, message, details) + details = "\nDetails:\n" + details += "\n".join(f"{label}: {info}" for label, info in self.details.items()) + return "".join([doc, message, details]) class KeaError(KeaException): """Kea failed during command processing""" From 9b77b90fdddc19695fc4392fdb276c288903ed87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:28:10 +0200 Subject: [PATCH 66/77] Add the ability to supply response attributes in our HTTP mock what: before this change, our mocking functionality only allowed for setting the string of the response object that is returned by requests.post() and requests.Session().post(), by using responsequeue.add("", ""). responsequeue.autofill("dhcp<4 or 6>", config_to_return, statistics_to_return) This commit modifies adds an extra parameter to both of these functions, the `attrs` parameter: responsequeue.add("", "", attrs={}). responsequeue.autofill("dhcp<4 or 6>", config_to_return, statistics_to_return, attrs={}) if `attrs` = {"myattr": "myval"}, then the requests.Response() object returned by any call to requests.post() or requests.Session().post() will have the attribute "myattr": attrs = {"myattr", "myval"} responsequeue.add("", "", attrs) response = requests.post(...) assert response.myattr == "myval" --- tests/unittests/dhcp/kea_metrics_test.py | 91 ++++++++++++++---------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 92d1a7f70b..de07f385f1 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -15,7 +15,7 @@ def test_dhcp6_config_and_statistic_response_that_is_valid_should_return_every_m valid_dhcp6, responsequeue ): config, statistics, expected_metrics = valid_dhcp6 - responsequeue.prefill("dhcp6", config, statistics) + responsequeue.autofill("dhcp6", config, statistics) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=6, tzinfo=timezone.utc) assert set(source.fetch_metrics()) == set(expected_metrics) @@ -24,7 +24,7 @@ def test_dhcp4_config_and_statistic_response_that_is_valid_should_return_every_m valid_dhcp4, responsequeue ): config, statistics, expected_metrics = valid_dhcp4 - responsequeue.prefill("dhcp4", config, statistics) + responsequeue.autofill("dhcp4", config, statistics) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4, tzinfo=timezone.utc) assert set(source.fetch_metrics()) == set(expected_metrics) @@ -40,7 +40,7 @@ def test_config_response_with_error_status_should_raise_KeaError( fetch_metrics(), we cannot continue further, so we fail. """ config, statistics, _ = valid_dhcp4 - responsequeue.prefill("dhcp4", None, statistics) + responsequeue.autofill("dhcp4", None, statistics) responsequeue.add("config-get", kearesponse(config, status=status)) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4) with pytest.raises(KeaException): @@ -65,14 +65,14 @@ def test_any_response_with_invalid_format_should_raise_KeaError( responsequeue.clear() - responsequeue.prefill("dhcp4", config, None) + responsequeue.autofill("dhcp4", config, None) responsequeue.add("statistic-get", "{}") with pytest.raises(KeaException): source.fetch_metrics() responsequeue.clear() - responsequeue.prefill("dhcp4", None, statistics) + responsequeue.autofill("dhcp4", None, statistics) responsequeue.add("config-get", "{}") with pytest.raises(KeaException): source.fetch_metrics() @@ -82,7 +82,7 @@ def test_any_response_with_invalid_format_should_raise_KeaError( # config-hash-get is only called if some config-get includes a hash we can compare # with the next time we're attempting to fetch a config: config["Dhcp4"]["hash"] = "foo" - responsequeue.prefill("dhcp4", config, statistics) + responsequeue.autofill("dhcp4", config, statistics) responsequeue.add("config-hash-get", "{}") with pytest.raises(KeaException): source.fetch_metrics() @@ -99,14 +99,14 @@ def test_all_responses_is_empty_but_valid_should_yield_no_metrics( correct thing to do is to return an empty iterable. """ config, statistics, _ = valid_dhcp4 - responsequeue.prefill("dhcp4", None, statistics) + responsequeue.autofill("dhcp4", None, statistics) responsequeue.add("config-get", lambda **_: kearesponse({"Dhcp4": {}})) source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4, tzinfo=timezone.utc) assert list(source.fetch_metrics()) == [] responsequeue.clear() - responsequeue.prefill("dhcp4", config, None) + responsequeue.autofill("dhcp4", config, None) responsequeue.add( "statistic-get", lambda arguments, **_: kearesponse({arguments["name"]: []}) ) @@ -114,7 +114,7 @@ def test_all_responses_is_empty_but_valid_should_yield_no_metrics( responsequeue.clear() - responsequeue.prefill("dhcp4", config, None) + responsequeue.autofill("dhcp4", config, None) responsequeue.add("statistic-get", lambda **_: kearesponse({})) assert list(source.fetch_metrics()) == [] @@ -525,7 +525,6 @@ def kearesponse(val, status=KeaStatus.SUCCESS): ] ''' - @pytest.fixture(autouse=True) def responsequeue(monkeypatch): """ @@ -542,6 +541,31 @@ def responsequeue(monkeypatch): responsequeue.clear() can be used to clear the all fifo queues. """ + """ + Any test that include this fixture, will automatically mock + `requests.Session.post()` and `requests.post()` (it uses the + set_response_handler fixture to set the response handler for + these mocked post() functions, so don't set this manually if this + fixture is used). + + This fixture returns a namespace with three functions: + + `responsequeue.add(command, text_or_func)`: add `text_or_func` to + the queue of text values to be set on the Response objects returned + by a post() call for the Kea command `command`. Any queue for a command `command` that is empty (the default) returns a Kea "command not supported" response. + if `text_or_func` is a string, it is added to the back of this queue and + becomes the text value of the response when it reaches the front of queue Then it gets popped off the queue. + if `text_or_func` is a callable, it is + added to the back of this queue. When it reaches the front of the queue, it is called + with the arguments that is post()'ed along with the Kea command `command`, and the return value becomes the + text value of the response. It is never be popped off the queue. + + `responsequeue.clear()`: Empty the queue for all commands. + + `responsequeue.autofill(service, config, statistics)`: fill the queue for the "config-get" and "statistic-get" + commands to mimic the response texts actually sent by a Kea Control Agent for a Kea DHCP server named `service` ("dhcp4" for ipv4 DHCP "dhcp6" for ipv6 DHCP) + that returns `config` on a "config-get" command and `statistics` on a "statistic-get-all" command. + """ # Dictonary of fifo queues, keyed by command name. A queue stored with key K has the textual content of the responses we want to return (in fifo order, one per call) on a call to requests.post with data that represents a Kea Control Agent command K command_responses = {} unknown_command_response = """[ @@ -573,64 +597,59 @@ def new_post_function(url, *args, data="{}", **kwargs): fifo = command_responses.get(command, deque()) if fifo: - first = fifo[0] - if callable(first): - text = first( - arguments=data.get("arguments", {}), service=data.get("service", []) - ) + next_text, attrs = fifo[0] + if callable(next_text): + arguments = data.get("arguments", {}) + service = data.get("service", []) + next_text = next_text(arguments=arguments, service=service) else: - text = str(first) + next_text = str(next_text) fifo.popleft() else: - text = unknown_command_response.format(command) + next_text = unknown_command_response.format(command) response = requests.Response() - response._content = text.encode("utf8") + response._content = next_text.encode("utf8") response.encoding = "utf8" - response.status_code = 400 + response.status_code = 200 response.reason = "OK" response.headers = kwargs.get("headers", {}) response.cookies = kwargs.get("cookies", {}) response.url = url response.close = lambda: True + + for attr, value in attrs.items(): + setattr(response, attr, value) + return response def new_post_method(self, url, *args, **kwargs): return new_post_function(url, *args, **kwargs) - def add_command_response(command_name, text): + def add_command_response(command_name, text, attrs={}): command_responses.setdefault(command_name, deque()) - command_responses[command_name].append(text) - - def remove_command_responses(command_name): - command_responses.pop(command_name, None) + command_responses[command_name].append((text, attrs)) def clear_command_responses(): command_responses.clear() - def prefill_command_responses(expected_service, config=None, statistics=None): + def autofill_command_responses(expected_service, config=None, statistics=None, attrs={}): def config_get_response(arguments, service): - assert service == [ - expected_service - ], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" return kearesponse(config) - def statistic_get_response(arguments, service): - assert service == [ - expected_service - ], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" return kearesponse({arguments["name"]: statistics[arguments["name"]]}) if config is not None: - add_command_response("config-get", config_get_response) + add_command_response("config-get", config_get_response, attrs) if statistics is not None: - add_command_response("statistic-get", statistic_get_response) + add_command_response("statistic-get", statistic_get_response, attrs) class ResponseQueue: add = add_command_response - remove = remove_command_responses clear = clear_command_responses - prefill = prefill_command_responses + autofill = autofill_command_responses monkeypatch.setattr(requests, 'post', new_post_function) monkeypatch.setattr(requests.Session, 'post', new_post_method) From 3dd847294b9b588a6ad419c752d82e33c4f317cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:39:35 +0200 Subject: [PATCH 67/77] Add test that assures that KeaException bubbles up on any HTTP error --- tests/unittests/dhcp/kea_metrics_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index de07f385f1..18510ce4de 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -119,6 +119,18 @@ def test_all_responses_is_empty_but_valid_should_yield_no_metrics( assert list(source.fetch_metrics()) == [] +def test_response_with_http_error_status_code_should_cause_KeaException_to_be_raised( + valid_dhcp4, responsequeue +): + config, statistics, _ = valid_dhcp4 + responsequeue.autofill("dhcp4", config, statistics, attrs={"status_code": 403}) + + source = KeaDhcpMetricSource("192.0.1.2", 80, dhcp_version=4, tzinfo=timezone.utc) + + with pytest.raises(KeaException): + source.fetch_metrics() + + @pytest.fixture def valid_dhcp6(): config = { From 8d5281bbdffe3e6d103428d8983853f6489d3879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 26 Jul 2024 15:43:32 +0200 Subject: [PATCH 68/77] Fix linting errors --- python/nav/dhcp/generic_metrics.py | 7 +++-- python/nav/dhcp/kea_metrics.py | 34 ++++++++++++++---------- tests/unittests/dhcp/kea_metrics_test.py | 16 ++++++++--- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index c5d32c0f8b..ad31daa8de 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -10,8 +10,9 @@ class DhcpMetricKey(Enum): TOTAL = "total" # total addresses managed by dhcp ASSIGNED = "assigned" # assigned addresses + def __str__(self): - return self.value # graphite key + return self.value # graphite key @dataclass(frozen=True) @@ -42,9 +43,7 @@ def fetch_metrics(self) -> Iterator[DhcpMetric]: raise NotImplementedError def fetch_metrics_to_graphite( - self, - host=CONFIG.get("carbon", "host"), - port=CONFIG.getint("carbon", "port") + self, host=CONFIG.get("carbon", "host"), port=CONFIG.getint("carbon", "port") ): """ Fetch metrics describing total amount of addresses diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 878dc28923..a9de75d1d5 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -13,6 +13,7 @@ _logger = logging.getLogger(__name__) + class KeaDhcpMetricSource(DhcpMetricSource): """ Communicates with a Kea Control Agent and fetches metrics for each @@ -105,7 +106,6 @@ def fetch_metrics(self) -> list[DhcpMetric]: ) metrics.append(metric) - newsubnets = _subnets_of_config(self._fetch_config(s), self.dhcp_version) if sorted(subnets) != sorted(newsubnets): _logger.error( @@ -116,7 +116,6 @@ def fetch_metrics(self) -> list[DhcpMetric]: return metrics - def _fetch_config(self, session: requests.Session) -> dict: """ Returns the current config of the Kea DHCP server that the Kea @@ -133,7 +132,7 @@ def _fetch_config(self, session: requests.Session) -> dict: except KeyError as err: raise KeaException( "Unrecognizable response to the 'config-get' request", - {"Response": response} + {"Response": response}, ) from err return self.dhcp_config @@ -152,7 +151,6 @@ def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: _logger.debug(str(err)) return None - def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict: """ Returns the response from the Kea Control Agent to the query @@ -199,19 +197,17 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict raise KeaException( "Server does not look like a Kea Control Agent; " "expected response content to be JSON", - request_summary + request_summary, ) from err except RequestException as err: raise KeaException( - "HTTP-related error during request to server", - request_summary + "HTTP-related error during request to server", request_summary ) from err if not isinstance(responses, list): # See https://kea.readthedocs.io/en/kea-2.6.0/arm/ctrl-channel.html#control-agent-command-response-format raise KeaException( - "Invalid response; server has likely rejected a query", - request_summary + "Invalid response; server has likely rejected a query", request_summary ) if not (len(responses) == 1 and "result" in responses[0]): # `post-data` queries *one* service. Thus `responses` should contain *one* response. @@ -219,7 +215,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict "Server does not look like a Kea Control Agent; " "expected response content to be a JSON list " "of a single object that has 'result' as one of its keys. ", - request_summary + request_summary, ) request_summary["Validity"] = "Valid Kea response" @@ -240,7 +236,6 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict raise KeaConflict(details=request_summary) raise KeaException("Kea returned an unkown status response", request_summary) - def _parsetime(self, timestamp: str) -> int: """Parse the timestamp string used in Kea's timeseries into unix time""" fmt = "%Y-%m-%d %H:%M:%S.%f" @@ -262,8 +257,7 @@ def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: prefix = subnet.get("subnet", None) if id is None or prefix is None: _logger.warning( - "id or prefix missing from a subnet's configuration: %r", - subnet + "id or prefix missing from a subnet's configuration: %r", subnet ) continue subnets.append((id, IP(prefix))) @@ -272,6 +266,7 @@ def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: class KeaException(GeneralException): """Error related to interaction with a Kea Control Agent""" + def __init__(self, message: str = "", details: dict[str, str] = {}): self.message = message self.details = details @@ -284,20 +279,31 @@ def __str__(self) -> str: message = f": {self.message}" if self.details: details = "\nDetails:\n" - details += "\n".join(f"{label}: {info}" for label, info in self.details.items()) + details += "\n".join( + f"{label}: {info}" for label, info in self.details.items() + ) return "".join([doc, message, details]) + class KeaError(KeaException): """Kea failed during command processing""" + + class KeaUnsupported(KeaException): """Unsupported command""" + + class KeaEmpty(KeaException): """Requested resource not found""" + + class KeaConflict(KeaException): """Kea failed to apply requested changes due to conflicts with its server state""" + class KeaStatus(IntEnum): """Status of a response sent from a Kea Control Agent""" + SUCCESS = 0 ERROR = 1 UNSUPPORTED = 2 diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 18510ce4de..39868d5768 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -120,7 +120,7 @@ def test_all_responses_is_empty_but_valid_should_yield_no_metrics( def test_response_with_http_error_status_code_should_cause_KeaException_to_be_raised( - valid_dhcp4, responsequeue + valid_dhcp4, responsequeue ): config, statistics, _ = valid_dhcp4 responsequeue.autofill("dhcp4", config, statistics, attrs={"status_code": 403}) @@ -537,6 +537,7 @@ def kearesponse(val, status=KeaStatus.SUCCESS): ] ''' + @pytest.fixture(autouse=True) def responsequeue(monkeypatch): """ @@ -645,12 +646,19 @@ def add_command_response(command_name, text, attrs={}): def clear_command_responses(): command_responses.clear() - def autofill_command_responses(expected_service, config=None, statistics=None, attrs={}): + def autofill_command_responses( + expected_service, config=None, statistics=None, attrs={} + ): def config_get_response(arguments, service): - assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + assert service == [ + expected_service + ], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" return kearesponse(config) + def statistic_get_response(arguments, service): - assert service == [expected_service], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" + assert service == [ + expected_service + ], f"KeaDhcpSource for service [{expected_service}] should not send requests to {service}" return kearesponse({arguments["name"]: statistics[arguments["name"]]}) if config is not None: From df1808097c7602522927115855687fecf501c7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Mon, 29 Jul 2024 10:26:20 +0200 Subject: [PATCH 69/77] Declare and use dhcp metric path in nav.metrics.templates --- python/nav/dhcp/generic_metrics.py | 14 +++++--------- python/nav/metrics/templates.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index ad31daa8de..824aeeb6de 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -2,7 +2,7 @@ from enum import Enum from IPy import IP from nav.metrics import carbon, CONFIG -from nav.metrics.names import escape_metric_name +from nav.metrics.templates import metric_path_for_subnet_dhcp from typing import Iterator from datetime import datetime @@ -29,12 +29,6 @@ class DhcpMetricSource: specific line of DHCP servers and import the metrics into NAV's graphite server. Subclasses need to implement `fetch_metrics`. """ - - graphite_prefix: str - - def __init__(self, graphite_prefix="nav.dhcp"): - self.graphite_prefix = graphite_prefix - def fetch_metrics(self) -> Iterator[DhcpMetric]: """ Fetch DhcpMetrics having keys `TOTAL` and `ASSIGNED` for each subnet of the @@ -54,7 +48,9 @@ def fetch_metrics_to_graphite( """ graphite_metrics = [] for metric in self.fetch_metrics(): - graphite_path = f"{self.graphite_prefix}.{escape_metric_name(metric.subnet_prefix.strNormal())}.{metric.key}" + metric_path = metric_path_for_subnet_dhcp( + metric.subnet_prefix, str(metric.key) + ) datapoint = (metric.timestamp, metric.value) - graphite_metrics.append((graphite_path, datapoint)) + graphite_metrics.append((metric_path, datapoint)) carbon.send_metrics_to(graphite_metrics, host, port) diff --git a/python/nav/metrics/templates.py b/python/nav/metrics/templates.py index 0f8a70e2a9..9207725804 100644 --- a/python/nav/metrics/templates.py +++ b/python/nav/metrics/templates.py @@ -184,3 +184,13 @@ def metric_path_for_multicast_usage(group, sysname): group=metric_prefix_for_multicast_group(group), sysname=escape_metric_name(sysname), ) + + +def metric_path_for_subnet_dhcp(subnet_prefix, metric_name): + tmpl = "nav.dhcp.{subnet_prefix}.{metric_name}" + if hasattr(subnet_prefix, 'strNormal') and callable(subnet_prefix.strNormal): + subnet_prefix = subnet_prefix.strNormal() + return tmpl.format( + subnet_prefix=escape_metric_name(subnet_prefix), + metric_name=metric_name + ) From 7322f4f37f96bdf400aaff3cae70bba74a0c716d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 10:28:12 +0200 Subject: [PATCH 70/77] Change dhcp subnet metric path to include IP address and port --- python/nav/dhcp/generic_metrics.py | 22 +++++++++++++++++++--- python/nav/metrics/templates.py | 15 +++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 824aeeb6de..03a9b9b88c 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -2,7 +2,7 @@ from enum import Enum from IPy import IP from nav.metrics import carbon, CONFIG -from nav.metrics.templates import metric_path_for_subnet_dhcp +from nav.metrics.templates import metric_path_for_subnet_dhcp, metric_path_for_ipdev_subnet_dhcp from typing import Iterator from datetime import datetime @@ -28,7 +28,23 @@ class DhcpMetricSource: Superclass for all classes that wish to collect metrics from a specific line of DHCP servers and import the metrics into NAV's graphite server. Subclasses need to implement `fetch_metrics`. + + The `fetch_metrics` docstring specifies which metrics are + collected. """ + def __init__(self, address, port): + """ + All subclasses that collect metrics from a DHCP server must + specify the address and port of the server it collects metrics + from so that we can reliably discern the metrics collected + from different servers (it is for example possible that two + servers serve the same subnet and hence it is not viable to + discern the metrics collected solely based on the subnet it is + for). + """ + self.address = address + self.port = port + def fetch_metrics(self) -> Iterator[DhcpMetric]: """ Fetch DhcpMetrics having keys `TOTAL` and `ASSIGNED` for each subnet of the @@ -48,8 +64,8 @@ def fetch_metrics_to_graphite( """ graphite_metrics = [] for metric in self.fetch_metrics(): - metric_path = metric_path_for_subnet_dhcp( - metric.subnet_prefix, str(metric.key) + metric_path = metric_path_for_ipdev_subnet_dhcp( + metric.subnet_prefix, str(metric.key), self.address, self.port ) datapoint = (metric.timestamp, metric.value) graphite_metrics.append((metric_path, datapoint)) diff --git a/python/nav/metrics/templates.py b/python/nav/metrics/templates.py index 9207725804..8239011a6c 100644 --- a/python/nav/metrics/templates.py +++ b/python/nav/metrics/templates.py @@ -194,3 +194,18 @@ def metric_path_for_subnet_dhcp(subnet_prefix, metric_name): subnet_prefix=escape_metric_name(subnet_prefix), metric_name=metric_name ) + +def metric_path_for_ipdev_subnet_dhcp(subnet_prefix, metric_name, address, port): + """ + Metric path that is automatically shown in a netbox's system info tab + """ + tmpl = "nav.devices.{address}.dhcp.{port}.subnet.{subnet_prefix}.{metric_name}" + if hasattr(subnet_prefix, 'strNormal') and callable(subnet_prefix.strNormal): + subnet_prefix = subnet_prefix.strNormal() # canonical name for IPy.IP instances + if hasattr(address, 'strNormal') and callable(address.strNormal): + address = address.strNormal() # canonical name for IPy.IP instances + return tmpl.format( + address=escape_metric_name(address), + port=str(port), + subnet_prefix=escape_metric_name(subnet_prefix), + ) From 3fd40e0a59198dcc267456f54c942e970df956a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 10:29:38 +0200 Subject: [PATCH 71/77] Set http requests' content-type to JSON --- python/nav/dhcp/kea_metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index a9de75d1d5..2724fe29a4 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -188,6 +188,7 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict self.rest_uri, data=post_data, timeout=self.timeout, + headers={"Content-Type": "application/json"}, ) request_summary["Status"] = "complete" request_summary["HTTP Status"] = responses.status_code From 3fb60a4bde09e052a8219743071c6e0e9586e177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 10:31:09 +0200 Subject: [PATCH 72/77] Use numeric timestamp instead of datetime instances for dhcp metrics (this is what the graphite exporting function expects) --- python/nav/dhcp/kea_metrics.py | 4 +-- tests/unittests/dhcp/kea_metrics_test.py | 34 ++++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 2724fe29a4..f7fcfae262 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -237,10 +237,10 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict raise KeaConflict(details=request_summary) raise KeaException("Kea returned an unkown status response", request_summary) - def _parsetime(self, timestamp: str) -> int: + def _parsetime(self, timestamp: str) -> float: """Parse the timestamp string used in Kea's timeseries into unix time""" fmt = "%Y-%m-%d %H:%M:%S.%f" - return datetime.strptime(timestamp, fmt).replace(tzinfo=self.tzinfo) + return datetime.strptime(timestamp, fmt).replace(tzinfo=self.tzinfo).timestamp() def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: diff --git a/tests/unittests/dhcp/kea_metrics_test.py b/tests/unittests/dhcp/kea_metrics_test.py index 39868d5768..fd22295821 100644 --- a/tests/unittests/dhcp/kea_metrics_test.py +++ b/tests/unittests/dhcp/kea_metrics_test.py @@ -223,103 +223,103 @@ def valid_dhcp6(): expected_metrics = [ DhcpMetric( - datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00"), + datetime.fromisoformat("2024-07-22T09:06:58.140438+00:00").timestamp(), IP("2001:db8:1:1::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00"), + datetime.fromisoformat("2024-07-05T20:44:54.230608+00:00").timestamp(), IP("2001:db8:1:1::/64"), DhcpMetricKey.ASSIGNED, 0, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00"), + datetime.fromisoformat("2024-07-05T09:15:05.626594+00:00").timestamp(), IP("2001:db8:1:1::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00"), + datetime.fromisoformat("2024-07-03T16:13:59.401058+00:00").timestamp(), IP("2001:db8:1:1::/64"), DhcpMetricKey.TOTAL, 239, ), DhcpMetric( - datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00").timestamp(), IP("2001:db8:1:2::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00").timestamp(), IP("2001:db8:1:2::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00"), + datetime.fromisoformat("2024-07-05T09:15:05.626595+00:00").timestamp(), IP("2001:db8:1:2::/64"), DhcpMetricKey.ASSIGNED, 2, ), DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00").timestamp(), IP("2001:db8:1:2::/64"), DhcpMetricKey.TOTAL, 240, ), DhcpMetric( - datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00").timestamp(), IP("2001:db8:1:3::/64"), DhcpMetricKey.ASSIGNED, 4, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00").timestamp(), IP("2001:db8:1:3::/64"), DhcpMetricKey.ASSIGNED, 5, ), DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00").timestamp(), IP("2001:db8:1:3::/64"), DhcpMetricKey.TOTAL, 241, ), DhcpMetric( - datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00").timestamp(), IP("2001:db8:1:4::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00").timestamp(), IP("2001:db8:1:4::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00").timestamp(), IP("2001:db8:1:4::/64"), DhcpMetricKey.TOTAL, 242, ), DhcpMetric( - datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00"), + datetime.fromisoformat("2024-07-22T09:06:58.140439+00:00").timestamp(), IP("2001:db8:1:5::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00"), + datetime.fromisoformat("2024-07-05T20:44:54.230609+00:00").timestamp(), IP("2001:db8:1:5::/64"), DhcpMetricKey.ASSIGNED, 1, ), DhcpMetric( - datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00"), + datetime.fromisoformat("2024-07-03T16:13:59.401059+00:00").timestamp(), IP("2001:db8:1:5::/64"), DhcpMetricKey.TOTAL, 243, From 76da903d77c854e75a6c5a20555b26320dcf9e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 10:44:38 +0200 Subject: [PATCH 73/77] Make linter happy --- python/nav/dhcp/generic_metrics.py | 6 +++++- python/nav/metrics/templates.py | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 03a9b9b88c..4b5ee16e3b 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -2,7 +2,10 @@ from enum import Enum from IPy import IP from nav.metrics import carbon, CONFIG -from nav.metrics.templates import metric_path_for_subnet_dhcp, metric_path_for_ipdev_subnet_dhcp +from nav.metrics.templates import ( + metric_path_for_subnet_dhcp, + metric_path_for_ipdev_subnet_dhcp, +) from typing import Iterator from datetime import datetime @@ -32,6 +35,7 @@ class DhcpMetricSource: The `fetch_metrics` docstring specifies which metrics are collected. """ + def __init__(self, address, port): """ All subclasses that collect metrics from a DHCP server must diff --git a/python/nav/metrics/templates.py b/python/nav/metrics/templates.py index 8239011a6c..6611e79991 100644 --- a/python/nav/metrics/templates.py +++ b/python/nav/metrics/templates.py @@ -191,21 +191,21 @@ def metric_path_for_subnet_dhcp(subnet_prefix, metric_name): if hasattr(subnet_prefix, 'strNormal') and callable(subnet_prefix.strNormal): subnet_prefix = subnet_prefix.strNormal() return tmpl.format( - subnet_prefix=escape_metric_name(subnet_prefix), - metric_name=metric_name + subnet_prefix=escape_metric_name(subnet_prefix), metric_name=metric_name ) + def metric_path_for_ipdev_subnet_dhcp(subnet_prefix, metric_name, address, port): """ Metric path that is automatically shown in a netbox's system info tab """ tmpl = "nav.devices.{address}.dhcp.{port}.subnet.{subnet_prefix}.{metric_name}" if hasattr(subnet_prefix, 'strNormal') and callable(subnet_prefix.strNormal): - subnet_prefix = subnet_prefix.strNormal() # canonical name for IPy.IP instances + subnet_prefix = subnet_prefix.strNormal() # canonical name for IPy.IP instances if hasattr(address, 'strNormal') and callable(address.strNormal): - address = address.strNormal() # canonical name for IPy.IP instances + address = address.strNormal() # canonical name for IPy.IP instances return tmpl.format( address=escape_metric_name(address), port=str(port), subnet_prefix=escape_metric_name(subnet_prefix), - ) + ) From 8a3030e3b7376ccf5910fb80a36cda163f2e23d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 10:47:51 +0200 Subject: [PATCH 74/77] Update canonical dhcp metric path template's docstring --- python/nav/metrics/templates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/nav/metrics/templates.py b/python/nav/metrics/templates.py index 6611e79991..d72fa6a0e9 100644 --- a/python/nav/metrics/templates.py +++ b/python/nav/metrics/templates.py @@ -197,7 +197,8 @@ def metric_path_for_subnet_dhcp(subnet_prefix, metric_name): def metric_path_for_ipdev_subnet_dhcp(subnet_prefix, metric_name, address, port): """ - Metric path that is automatically shown in a netbox's system info tab + Metric path for dhcp metrics that will be automatically shown in a + netbox's 'System metrics' tab """ tmpl = "nav.devices.{address}.dhcp.{port}.subnet.{subnet_prefix}.{metric_name}" if hasattr(subnet_prefix, 'strNormal') and callable(subnet_prefix.strNormal): From 9b7dc94a1427ed7cdef1c4b49f9d838a4faf05c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 16:19:10 +0200 Subject: [PATCH 75/77] Enhance docstrings, using suggestions from PEP 257 --- python/nav/dhcp/kea_metrics.py | 44 ++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index f7fcfae262..7296e17945 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -1,3 +1,14 @@ +""" +Exports the KeaDhcpMetricSource class for fetching DHCP metrics from +Kea DHCP servers + | + Managed by NAV | Managed externally + | + HTTP | IPC +KeaDhcpMetricSource <---------> Kea Control Agent <=====> Kea DHCP4 server/Kea DHCP6 server + | + | +""" from IPy import IP from typing import Optional from itertools import chain @@ -16,10 +27,18 @@ class KeaDhcpMetricSource(DhcpMetricSource): """ - Communicates with a Kea Control Agent and fetches metrics for each - subnet managed by the Kea DHCP server serving a specific ip - version that is controlled by the Kea Control Agent (see - `KeaDhcpMetricSource.fetch_metrics`). + Communicates with a Kea Control Agent to enable fetching of DHCP + metrics for each subnet managed by some specific underlying Kea + DHCP4 or Kea DHCP6 server. + + The sole purpose of this class is to implement the superclass's + fetch_metrics() method. Public methods are: + + * fetch_metrics(): Fetches DHCP metrics for each subnet managed by + the Kea DHCP server. Metrics are returned as a list. + + * fetch_metrics_to_graphite(): Inherited from superclass. Fetches + DHCP metrics as above and sends these to a graphite server. """ def __init__( @@ -34,9 +53,13 @@ def __init__( **kwargs, ): """ - Returns a KeaDhcpMetricSource that fetches DHCP metrics via - the Kea Control Agent listening to `port` on `address`. + Instantiate a KeaDhcpMetricSource that fetches DHCP metrics + from the Kea DHCP server managing IP version `dhcp_version` + addresses, whose metrics is reachable via the Kea Control + Agent listening to `port` on `address`. + :param address: IP address of the Kea Control Agent + :param port: TCP port of the Kea Control Agent :param https: if True, use https. Otherwise, use http :param dhcp_version: ip version served by Kea DHCP server :param timeout: how long to wait for a http response from @@ -53,10 +76,11 @@ def __init__( def fetch_metrics(self) -> list[DhcpMetric]: """ - Returns a list containing the most recent DHCP metrics for - each subnet managed by the Kea DHCP server. For each subnet - and DhcpMetric-key combination, there is at least one - corresponding metric in the returned list if no errors occur. + Fetches and returns a list containing the most recent DHCP + metrics for each subnet managed by the Kea DHCP server. For + each subnet and DhcpMetric-key combination, there is at least + one corresponding metric in the returned list if no errors + occur. If the Kea Control Agent responds with an empty response to one or more of the requests for some metric(s), these metrics From 000f4dff0ef68aa360312ffeab49df5a3ecc8aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 16:21:46 +0200 Subject: [PATCH 76/77] Disinclude ip-address and tcp-port from graphite dhcp metric path --- python/nav/dhcp/generic_metrics.py | 22 +++------------------- python/nav/dhcp/kea_metrics.py | 16 +++++++--------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/python/nav/dhcp/generic_metrics.py b/python/nav/dhcp/generic_metrics.py index 4b5ee16e3b..e3f892745c 100644 --- a/python/nav/dhcp/generic_metrics.py +++ b/python/nav/dhcp/generic_metrics.py @@ -2,10 +2,7 @@ from enum import Enum from IPy import IP from nav.metrics import carbon, CONFIG -from nav.metrics.templates import ( - metric_path_for_subnet_dhcp, - metric_path_for_ipdev_subnet_dhcp, -) +from nav.metrics.templates import metric_path_for_subnet_dhcp from typing import Iterator from datetime import datetime @@ -36,19 +33,6 @@ class DhcpMetricSource: collected. """ - def __init__(self, address, port): - """ - All subclasses that collect metrics from a DHCP server must - specify the address and port of the server it collects metrics - from so that we can reliably discern the metrics collected - from different servers (it is for example possible that two - servers serve the same subnet and hence it is not viable to - discern the metrics collected solely based on the subnet it is - for). - """ - self.address = address - self.port = port - def fetch_metrics(self) -> Iterator[DhcpMetric]: """ Fetch DhcpMetrics having keys `TOTAL` and `ASSIGNED` for each subnet of the @@ -68,8 +52,8 @@ def fetch_metrics_to_graphite( """ graphite_metrics = [] for metric in self.fetch_metrics(): - metric_path = metric_path_for_ipdev_subnet_dhcp( - metric.subnet_prefix, str(metric.key), self.address, self.port + metric_path = metric_path_for_subnet_dhcp( + metric.subnet_prefix, str(metric.key) ) datapoint = (metric.timestamp, metric.value) graphite_metrics.append((metric_path, datapoint)) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 7296e17945..285881f697 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -45,12 +45,10 @@ def __init__( self, address: str, port: int, - *args, https: bool = True, dhcp_version: int = 4, timeout: int = 10, - tzinfo: datetime.tzinfo = datetime.now().astimezone().tzinfo, - **kwargs, + tzinfo: datetime.tzinfo = None, ): """ Instantiate a KeaDhcpMetricSource that fetches DHCP metrics @@ -66,13 +64,13 @@ def __init__( the Kea Control Agent before timing out :param tzinfo: the timezone of the Kea Control Agent. """ - super(*args, **kwargs) + super() scheme = "https" if https else "http" - self.rest_uri = f"{scheme}://{address}:{port}/" - self.dhcp_version = dhcp_version - self.dhcp_config: Optional[dict] = None - self.timeout = timeout - self.tzinfo = tzinfo + self._rest_uri = f"{scheme}://{address}:{port}/" + self._dhcp_version = dhcp_version + self._dhcp_config: Optional[dict] = None + self._timeout = timeout + self._tzinfo = tzinfo or datetime.now().astimezone().tzinfo def fetch_metrics(self) -> list[DhcpMetric]: """ From d3faa81753fe5b939340faa0e3c15ffe2af7100a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rund=20Helleb=C3=B8?= Date: Fri, 20 Sep 2024 16:22:47 +0200 Subject: [PATCH 77/77] Extract a fetch_subnet_metrics() help-func out from fetch_metrics() --- python/nav/dhcp/kea_metrics.py | 107 ++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 47 deletions(-) diff --git a/python/nav/dhcp/kea_metrics.py b/python/nav/dhcp/kea_metrics.py index 285881f697..3b16d5911b 100644 --- a/python/nav/dhcp/kea_metrics.py +++ b/python/nav/dhcp/kea_metrics.py @@ -24,6 +24,7 @@ _logger = logging.getLogger(__name__) +_SubnetTuple = tuple[int, IP] # (subnet_id, netprefix) class KeaDhcpMetricSource(DhcpMetricSource): """ @@ -97,66 +98,78 @@ def fetch_metrics(self) -> list[DhcpMetric]: General errors reported by the Kea Control Agent causes a KeaError to be raised. """ + metrics = [] + + with requests.Session() as session: + config = self._fetch_config(session) + subnets = _subnets_of_config(config, self._dhcp_version) + + for subnet in subnets: + subnet_metrics = self._fetch_subnet_metrics(subnet, session) + metrics.extend(subnet_metrics) + + newest_subnets = _subnets_of_config(self._fetch_config(session), self._dhcp_version) + if sorted(subnets) != sorted(newest_subnets): + _logger.warning( + "Subnet configuration was modified during DHCP metric fetching, " + "this may cause metric data being associated with wrong subnet." + ) + + return metrics + + + def _fetch_subnet_metrics( + self, subnet: _SubnetTuple, session: requests.Session + ) -> list[DhcpMetric]: metric_keys = ( ("total-addresses", DhcpMetricKey.TOTAL), ("assigned-addresses", DhcpMetricKey.ASSIGNED), ) metrics = [] - with requests.Session() as s: - config = self._fetch_config(s) - subnets = _subnets_of_config(config, self.dhcp_version) - for subnet_id, netprefix in subnets: - for kea_key, nav_key in metric_keys: - kea_name = f"subnet[{subnet_id}].{kea_key}" - try: - response = self._send_query(s, "statistic-get", name=kea_name) - except KeaEmpty: - continue - timeseries = response.get("arguments", {}).get(kea_name, []) - if len(timeseries) == 0: - _logger.error( - "fetch_metrics: Could not fetch metric '%r' for subnet " - "'%s' from Kea: '%s' from Kea is an empty list.", - nav_key, - netprefix, - kea_name, - ) - for value, timestring in timeseries: - metric = DhcpMetric( - self._parsetime(timestring), netprefix, nav_key, value - ) - metrics.append(metric) - - newsubnets = _subnets_of_config(self._fetch_config(s), self.dhcp_version) - if sorted(subnets) != sorted(newsubnets): - _logger.error( - "Subnet configuration was modified during metric fetching, " - "this may cause metric data being associated with wrong " - "subnet in some rare cases." - ) + for kea_key, nav_key in metric_keys: + kea_name = f"subnet[{subnet_id}].{kea_key}" + try: + response = self._send_query(session, "statistic-get", name=kea_name) + except KeaEmpty: + continue + timeseries = response.get("arguments", {}).get(kea_name, []) + if len(timeseries) == 0: + _logger.warning( + "Could not fetch metric '%r' for subnet '%s' from Kea: '%s' from Kea " + "is an empty list.", + nav_key, + netprefix, + kea_name, + ) + for value, timestring in timeseries: + metric = DhcpMetric( + self._parsetime(timestring), netprefix, nav_key, value + ) + metrics.append(metric) return metrics + def _fetch_config(self, session: requests.Session) -> dict: """ Returns the current config of the Kea DHCP server that the Kea Control Agent controls. """ if ( - self.dhcp_config is None - or (dhcp_confighash := self.dhcp_config.get("hash", None)) is None + self._dhcp_config is None + or (dhcp_confighash := self._dhcp_config.get("hash", None)) is None or self._fetch_config_hash(session) != dhcp_confighash ): response = self._send_query(session, "config-get") try: - self.dhcp_config = response["arguments"][f"Dhcp{self.dhcp_version}"] + self._dhcp_config = response["arguments"][f"Dhcp{self._dhcp_version}"] except KeyError as err: raise KeaException( "Unrecognizable response to the 'config-get' request", {"Response": response}, ) from err - return self.dhcp_config + return self._dhcp_config def _fetch_config_hash(self, session: requests.Session) -> Optional[str]: """ @@ -190,9 +203,9 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict to be raised. """ request_summary = { - "Description": f"Sending request to Kea Control Agent at {self.rest_uri}", + "Description": f"Sending request to Kea Control Agent at {self._rest_uri}", "Status": "sending", - "Location": self.rest_uri, + "Location": self._rest_uri, "Command": command, } _logger.debug(request_summary) @@ -201,15 +214,15 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict { "command": command, "arguments": {**kwargs}, - "service": [f"dhcp{self.dhcp_version}"], + "service": [f"dhcp{self._dhcp_version}"], } ) request_summary["Validity"] = "Invalid Kea response" try: responses = session.post( - self.rest_uri, + self._rest_uri, data=post_data, - timeout=self.timeout, + timeout=self._timeout, headers={"Content-Type": "application/json"}, ) request_summary["Status"] = "complete" @@ -262,10 +275,10 @@ def _send_query(self, session: requests.Session, command: str, **kwargs) -> dict def _parsetime(self, timestamp: str) -> float: """Parse the timestamp string used in Kea's timeseries into unix time""" fmt = "%Y-%m-%d %H:%M:%S.%f" - return datetime.strptime(timestamp, fmt).replace(tzinfo=self.tzinfo).timestamp() + return datetime.strptime(timestamp, fmt).replace(tzinfo=self._tzinfo).timestamp() -def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: +def _subnets_of_config(config: dict, ip_version: int) -> list[_SubnetTuple]: """ Returns a list containing one (subnet-id, subnet-prefix) tuple per subnet listed in the Kea DHCP configuration `config`. @@ -276,14 +289,14 @@ def _subnets_of_config(config: dict, ip_version: int) -> list[tuple[int, IP]]: [config.get(subnetkey, [])] + [network.get(subnetkey, []) for network in config.get("shared-networks", [])] ): - id = subnet.get("id", None) - prefix = subnet.get("subnet", None) - if id is None or prefix is None: + subnet_id = subnet.get("id", None) + netprefix = subnet.get("subnet", None) + if subnet_id is None or netprefix is None: _logger.warning( "id or prefix missing from a subnet's configuration: %r", subnet ) continue - subnets.append((id, IP(prefix))) + subnets.append((subnet_id, IP(netprefix))) return subnets