diff --git a/development/docker-compose.mysql.yml b/development/docker-compose.mysql.yml index 2f1103da..e1494749 100644 --- a/development/docker-compose.mysql.yml +++ b/development/docker-compose.mysql.yml @@ -18,8 +18,6 @@ services: - "development_mysql.env" db: image: "mysql:8" - command: - - "--max_connections=1000" env_file: - "development.env" - "creds.env" diff --git a/docs/admin/release_notes/version_2.1.md b/docs/admin/release_notes/version_2.1.md new file mode 100644 index 00000000..a2ac2dd4 --- /dev/null +++ b/docs/admin/release_notes/version_2.1.md @@ -0,0 +1,14 @@ + +# v2.1 Release Notes + +- Added support for XML Compliance. +- Hide Compliance tab if no compliance result exists. + + +### Added + +- [#1501](https://github.com/nautobot/nautobot-app-golden-config/issues/1501) - Add Support for XML Compliance + +### Fixed + +- [#723](https://github.com/nautobot/nautobot-app-golden-config/issues/723) - Hide compliance tab in device view if no compliance results exist. diff --git a/docs/admin/troubleshooting/E3031.md b/docs/admin/troubleshooting/E3031.md new file mode 100644 index 00000000..5364c17d --- /dev/null +++ b/docs/admin/troubleshooting/E3031.md @@ -0,0 +1,17 @@ +# E3031 Details + +## Message emitted: + +`E3031: Invalid XPath expression.` + +## Description: + +This error occurs when an invalid XPath expression is used in a Compliance Job, causing a `NornirNautobotException` to be raised. + +## Troubleshooting: + +Review the exception message and worker logs to determine the cause of the failure. + +## Recommendation: + +Ensure that you are using a valid XPath expression in the "Config to Match" section of your Compliance Rule. diff --git a/docs/images/compliance-rule-xml.png b/docs/images/compliance-rule-xml.png new file mode 100644 index 00000000..bda52728 Binary files /dev/null and b/docs/images/compliance-rule-xml.png differ diff --git a/docs/images/device-compliance-xml.png b/docs/images/device-compliance-xml.png new file mode 100644 index 00000000..9d9b1c30 Binary files /dev/null and b/docs/images/device-compliance-xml.png differ diff --git a/docs/user/app_feature_compliancexml.md b/docs/user/app_feature_compliancexml.md new file mode 100644 index 00000000..a339d85e --- /dev/null +++ b/docs/user/app_feature_compliancexml.md @@ -0,0 +1,54 @@ +# Navigating Compliance Using XML + +XML based compliance provides a mechanism to compliance check device configurations stored in XML format. + +## Defining Compliance Rules + +Compliance rules are defined as XML `config-type`. + +The `config to match` field is used to specify an XPath query. This query is used to select specific nodes in the XML configurations for comparison. If the `config to match` field is left blank, all nodes in the configurations will be compared. + +### XPath in Config to Match + +XPath (XML Path Language) is a query language for selecting nodes from an XML document. In our application, XPath is used in the `config to match` field to specify which parts of the device configurations should be compared. + +### Basic XPath Syntax + +Here is a quick reference for basic XPath syntax: + +| Expression | Description | +| --- | --- | +| `nodename` | Selects all nodes with the name "nodename" | +| `/` | Selects from the root node | +| `//` | Selects nodes in the document from the current node that match the selection no matter where they are | + +For more detailed information on XPath syntax, you can refer to the [Supported XPath syntax](https://docs.python.org/3/library/xml.etree.elementtree.html#supported-xpath-syntax). + +This NTC [blog](https://blog.networktocode.com/post/parsing-xml-with-python-and-ansible/) also covers XPath in more details. + +Here are some examples of XPath queries that can be used in the `config to match` field: + +![Example XML Compliance Rules](../images/compliance-rule-xml.png) + +## Device Config Compliance View + +![Config Compliance Device View](../images/device-compliance-xml.png) + +## Interpreting Diff Output + +The diff output shows the differences between the device configurations. Each line in the diff output represents a node in the XML configurations. The node is identified by its XPath, and the value of the node is shown after the comma. + +Here's a sample 'missing' output: + +```text +/config/system/aaa/user[1]/password[1], foo +/config/system/aaa/user[1]/role[1], admin +/config/system/aaa/radius/server[1]/host[1], 1.1.1.1 +/config/system/aaa/radius/server[1]/secret[1], foopass +/config/system/aaa/radius/server[2]/host[1], 2.2.2.2 +/config/system/aaa/radius/server[2]/secret[1], bazpass +``` + +This diff output represents the 'missing' portion when comparing the actual configuration to the intended configuration. Each line represents a node in the XML configuration that is presented in the intended configuration but is missing in the actual configuration. + +For example, the line `/config/system/aaa/user[1]/password[1], foo` indicates that the password node of the first user node under `/config/system/aaa` is expected to have a value of `foo` in the actual configuration. If this line appears in the diff output, it means this value is missing in the actual configuration. diff --git a/mkdocs.yml b/mkdocs.yml index 6fa13912..fca1c9dd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -155,6 +155,11 @@ nav: - E3024: "admin/troubleshooting/E3024.md" - E3025: "admin/troubleshooting/E3025.md" - E3026: "admin/troubleshooting/E3026.md" + - E3027: "admin/troubleshooting/E3027.md" + - E3028: "admin/troubleshooting/E3028.md" + - E3029: "admin/troubleshooting/E3029.md" + - E3030: "admin/troubleshooting/E3030.md" + - E3031: "admin/troubleshooting/E3031.md" - Migrating To v2: "admin/migrating_to_v2.md" - Release Notes: - "admin/release_notes/index.md" diff --git a/nautobot_golden_config/choices.py b/nautobot_golden_config/choices.py index 612b3203..7531b279 100644 --- a/nautobot_golden_config/choices.py +++ b/nautobot_golden_config/choices.py @@ -7,10 +7,12 @@ class ComplianceRuleConfigTypeChoice(ChoiceSet): TYPE_CLI = "cli" TYPE_JSON = "json" + TYPE_XML = "xml" CHOICES = ( (TYPE_CLI, "CLI"), (TYPE_JSON, "JSON"), + (TYPE_XML, "XML"), ) diff --git a/nautobot_golden_config/models.py b/nautobot_golden_config/models.py index a9239427..924f5158 100644 --- a/nautobot_golden_config/models.py +++ b/nautobot_golden_config/models.py @@ -15,6 +15,8 @@ from nautobot.extras.models.statuses import StatusField from nautobot.extras.utils import extras_features from netutils.config.compliance import feature_compliance +from xmldiff import main, actions + from nautobot_golden_config.choices import ComplianceRuleConfigTypeChoice, ConfigPlanTypeChoice, RemediationTypeChoice from nautobot_golden_config.utilities.constant import ENABLE_SOTAGG, PLUGIN_CFG @@ -118,6 +120,41 @@ def _normalize_diff(diff, path_to_diff): } +def _get_xml_compliance(obj): + """This function performs the actual compliance for xml serializable data.""" + + def _normalize_diff(diff): + """Format the diff output to a list of nodes with values that have updated.""" + formatted_diff = [] + for operation in diff: + if isinstance(operation, actions.UpdateTextIn): + formatted_operation = f"{operation.node}, {operation.text}" + formatted_diff.append(formatted_operation) + return "\n".join(formatted_diff) + + # Options for the diff operation. These are set to prefer updates over node insertions/deletions. + diff_options = { + "F": 0.1, + "fast_match": True, + } + missing = main.diff_texts(obj.actual, obj.intended, diff_options=diff_options) + extra = main.diff_texts(obj.intended, obj.actual, diff_options=diff_options) + + compliance = not missing and not extra + compliance_int = int(compliance) + ordered = obj.ordered + missing = _null_to_empty(_normalize_diff(missing)) + extra = _null_to_empty(_normalize_diff(extra)) + + return { + "compliance": compliance, + "compliance_int": compliance_int, + "ordered": ordered, + "missing": missing, + "extra": extra, + } + + def _verify_get_custom_compliance_data(compliance_details): """This function verifies the data is as expected when a custom function is used.""" for val in ["compliance", "compliance_int", "ordered", "missing", "extra"]: @@ -171,6 +208,7 @@ def _get_hierconfig_remediation(obj): FUNC_MAPPER = { ComplianceRuleConfigTypeChoice.TYPE_CLI: _get_cli_compliance, ComplianceRuleConfigTypeChoice.TYPE_JSON: _get_json_compliance, + ComplianceRuleConfigTypeChoice.TYPE_XML: _get_xml_compliance, RemediationTypeChoice.TYPE_HIERCONFIG: _get_hierconfig_remediation, } # The below conditionally add the custom provided compliance type @@ -249,13 +287,13 @@ class ComplianceRule(PrimaryModel): # pylint: disable=too-many-ancestors match_config = models.TextField( blank=True, verbose_name="Config to Match", - help_text="The config to match that is matched based on the parent most configuration. E.g.: For CLI `router bgp` or `ntp`. For JSON this is a top level key name.", + help_text="The config to match that is matched based on the parent most configuration. E.g.: For CLI `router bgp` or `ntp`. For JSON this is a top level key name. For XML this is a xpath query.", ) config_type = models.CharField( max_length=20, default=ComplianceRuleConfigTypeChoice.TYPE_CLI, choices=ComplianceRuleConfigTypeChoice, - help_text="Whether the configuration is in CLI or JSON/structured format.", + help_text="Whether the configuration is in CLI, JSON, or XML format.", ) custom_compliance = models.BooleanField( default=False, help_text="Whether this Compliance Rule is proceeded as custom." diff --git a/nautobot_golden_config/nornir_plays/config_compliance.py b/nautobot_golden_config/nornir_plays/config_compliance.py index 0e354fc1..1a8f0edd 100644 --- a/nautobot_golden_config/nornir_plays/config_compliance.py +++ b/nautobot_golden_config/nornir_plays/config_compliance.py @@ -6,8 +6,9 @@ import os from collections import defaultdict from datetime import datetime - from django.utils.timezone import make_aware +from lxml import etree # nosec + from nautobot_plugin_nornir.constants import NORNIR_SETTINGS from nautobot_plugin_nornir.plugins.inventory.nautobot_orm import NautobotORMInventory from netutils.config.compliance import _open_file_config, parser_map, section_config @@ -20,8 +21,14 @@ from nautobot_golden_config.models import ComplianceRule, ConfigCompliance, GoldenConfig from nautobot_golden_config.nornir_plays.processor import ProcessGoldenConfig from nautobot_golden_config.utilities.db_management import close_threaded_db_connections -from nautobot_golden_config.utilities.helper import get_json_config, render_jinja_template, verify_settings from nautobot_golden_config.utilities.logger import NornirLogger +from nautobot_golden_config.utilities.helper import ( + get_json_config, + get_xml_config, + render_jinja_template, + verify_settings, + get_xml_subtree_with_full_path, +) InventoryPluginRegister.register("nautobot-inventory", NautobotORMInventory) LOGGER = logging.getLogger(__name__) @@ -64,6 +71,24 @@ def get_config_element(rule, config, obj, logger): else: config_element = config_json + elif rule["obj"].config_type == ComplianceRuleConfigTypeChoice.TYPE_XML: + config_xml = get_xml_config(config) + + if not config_xml: + error_msg = "`E3002:` Unable to interpret configuration as XML." + logger.error(error_msg, extra={"object": obj}) + raise NornirNautobotException(error_msg) + + if rule["obj"].match_config: + try: + config_element = get_xml_subtree_with_full_path(config_xml, rule["obj"].match_config) + except etree.XPathError as err: + error_msg = f"`E3031:` Invalid XPath expression - `{rule['obj'].match_config}`" + logger.error(error_msg, extra={"object": obj}) + raise NornirNautobotException(error_msg) from err + else: + config_element = etree.tostring(config_xml, encoding="unicode", pretty_print=True) + elif rule["obj"].config_type == ComplianceRuleConfigTypeChoice.TYPE_CLI: if obj.platform.network_driver_mappings["netutils_parser"] not in parser_map: error_msg = f"`E3003:` There is currently no CLI-config parser support for platform network_driver `{obj.platform.network_driver}`, preemptively failed." diff --git a/nautobot_golden_config/template_content.py b/nautobot_golden_config/template_content.py index 04f71628..ff97ce05 100644 --- a/nautobot_golden_config/template_content.py +++ b/nautobot_golden_config/template_content.py @@ -12,18 +12,19 @@ class ConfigComplianceDeviceCheck(PluginTemplateExtension): # pylint: disable=a model = "dcim.device" - def get_device(self): - """Get device object.""" + @property + def device(self): + """Device presented in detail view.""" return self.context["object"] def right_page(self): """Content to add to the configuration compliance.""" - comp_obj = ConfigCompliance.objects.filter(device=self.get_device()).values("rule__feature__name", "compliance") + comp_obj = ConfigCompliance.objects.filter(device=self.device).values("rule__feature__name", "compliance") if not comp_obj: return "" extra_context = { "compliance": comp_obj, - "device": self.get_device(), + "device": self.device, "template_type": "devicetab", } return self.render( @@ -34,18 +35,21 @@ def right_page(self): def detail_tabs(self): """Add a Configuration Compliance tab to the Device detail view if the Configuration Compliance associated to it.""" try: - return [ - { - "title": "Configuration Compliance", - "url": reverse( - "plugins:nautobot_golden_config:configcompliance_devicetab", - kwargs={"pk": self.get_device().pk}, - ), - } - ] + if ConfigCompliance.objects.filter(device=self.device): + return [ + { + "title": "Configuration Compliance", + "url": reverse( + "plugins:nautobot_golden_config:configcompliance_devicetab", + kwargs={"pk": self.device.pk}, + ), + } + ] except ObjectDoesNotExist: return [] + return [] + class ConfigComplianceLocationCheck(PluginTemplateExtension): # pylint: disable=abstract-method """App extension class for config compliance.""" diff --git a/nautobot_golden_config/templates/nautobot_golden_config/configcompliance_devicetab.html b/nautobot_golden_config/templates/nautobot_golden_config/configcompliance_devicetab.html index a8e0839a..e42c4a69 100644 --- a/nautobot_golden_config/templates/nautobot_golden_config/configcompliance_devicetab.html +++ b/nautobot_golden_config/templates/nautobot_golden_config/configcompliance_devicetab.html @@ -111,7 +111,13 @@ Configuration - 
{{ item.actual|placeholder|condition_render_json }}
 + {% if item.rule.config_type == "xml" %} + 
{{ item.actual|placeholder }}
 + {% elif item.rule.config_type == "json" %} + 
{{ item.actual|placeholder|condition_render_json }}
 + {% else %} + 
{{ item.actual|placeholder }}
 + {% endif %}