Merge pull request #767 from nautobot/release-2.1

Release 2.1

development/docker-compose.mysql.yml

@@ -18,8 +18,6 @@ services:
       - "development_mysql.env"
   db:
     image: "mysql:8"
-    command:
-      - "--max_connections=1000"
     env_file:
       - "development.env"
       - "creds.env"

docs/admin/release_notes/version_2.1.md

@@ -0,0 +1,14 @@
+
+# v2.1 Release Notes
+
+- Added support for XML Compliance.
+- Hide Compliance tab if no compliance result exists.
+
+
+### Added
+
+- [#1501](https://github.com/nautobot/nautobot-app-golden-config/issues/1501) - Add Support for XML Compliance
+
+### Fixed
+
+- [#723](https://github.com/nautobot/nautobot-app-golden-config/issues/723) - Hide compliance tab in device view if no compliance results exist.

docs/admin/troubleshooting/E3031.md

@@ -0,0 +1,17 @@
+# E3031 Details
+
+## Message emitted:
+
+`E3031: Invalid XPath expression.`
+
+## Description:
+
+This error occurs when an invalid XPath expression is used in a Compliance Job, causing a `NornirNautobotException` to be raised.
+
+## Troubleshooting:
+
+Review the exception message and worker logs to determine the cause of the failure.
+
+## Recommendation:
+
+Ensure that you are using a valid XPath expression in the "Config to Match" section of your Compliance Rule.

docs/user/app_feature_compliancexml.md

@@ -0,0 +1,54 @@
+# Navigating Compliance Using XML
+
+XML based compliance provides a mechanism to compliance check device configurations stored in XML format.
+
+## Defining Compliance Rules
+
+Compliance rules are defined as XML `config-type`.
+
+The `config to match` field is used to specify an XPath query. This query is used to select specific nodes in the XML configurations for comparison. If the `config to match` field is left blank, all nodes in the configurations will be compared.
+
+### XPath in Config to Match
+
+XPath (XML Path Language) is a query language for selecting nodes from an XML document. In our application, XPath is used in the `config to match` field to specify which parts of the device configurations should be compared.
+
+### Basic XPath Syntax
+
+Here is a quick reference for basic XPath syntax:
+
+| Expression | Description |
+| --- | --- |
+| `nodename` | Selects all nodes with the name "nodename" |
+| `/` | Selects from the root node |
+| `//` | Selects nodes in the document from the current node that match the selection no matter where they are |
+
+For more detailed information on XPath syntax, you can refer to the [Supported XPath syntax](https://docs.python.org/3/library/xml.etree.elementtree.html#supported-xpath-syntax).
+
+This NTC [blog](https://blog.networktocode.com/post/parsing-xml-with-python-and-ansible/) also covers XPath in more details.
+
+Here are some examples of XPath queries that can be used in the `config to match` field:
+
+![Example XML Compliance Rules](../images/compliance-rule-xml.png)
+
+## Device Config Compliance View
+
+![Config Compliance Device View](../images/device-compliance-xml.png)
+
+## Interpreting Diff Output
+
+The diff output shows the differences between the device configurations. Each line in the diff output represents a node in the XML configurations. The node is identified by its XPath, and the value of the node is shown after the comma.
+
+Here's a sample 'missing' output:
+
+```text
+/config/system/aaa/user[1]/password[1], foo
+/config/system/aaa/user[1]/role[1], admin
+/config/system/aaa/radius/server[1]/host[1], 1.1.1.1
+/config/system/aaa/radius/server[1]/secret[1], foopass
+/config/system/aaa/radius/server[2]/host[1], 2.2.2.2
+/config/system/aaa/radius/server[2]/secret[1], bazpass
+```
+
+This diff output represents the 'missing' portion when comparing the actual configuration to the intended configuration. Each line represents a node in the XML configuration that is presented in the intended configuration but is missing in the actual configuration.
+
+For example, the line `/config/system/aaa/user[1]/password[1], foo` indicates that the password node of the first user node under `/config/system/aaa` is expected to have a value of `foo` in the actual configuration. If this line appears in the diff output, it means this value is missing in the actual configuration.

mkdocs.yml

@@ -155,6 +155,11 @@ nav:
           - E3024: "admin/troubleshooting/E3024.md"
           - E3025: "admin/troubleshooting/E3025.md"
           - E3026: "admin/troubleshooting/E3026.md"
+          - E3027: "admin/troubleshooting/E3027.md"
+          - E3028: "admin/troubleshooting/E3028.md"
+          - E3029: "admin/troubleshooting/E3029.md"
+          - E3030: "admin/troubleshooting/E3030.md"
+          - E3031: "admin/troubleshooting/E3031.md"
       - Migrating To v2: "admin/migrating_to_v2.md"
       - Release Notes:
           - "admin/release_notes/index.md"

nautobot_golden_config/choices.py

@@ -7,10 +7,12 @@ class ComplianceRuleConfigTypeChoice(ChoiceSet):
 
     TYPE_CLI = "cli"
     TYPE_JSON = "json"
+    TYPE_XML = "xml"
 
     CHOICES = (
         (TYPE_CLI, "CLI"),
         (TYPE_JSON, "JSON"),
+        (TYPE_XML, "XML"),
     )
 
 

nautobot_golden_config/models.py

@@ -15,6 +15,8 @@
 from nautobot.extras.models.statuses import StatusField
 from nautobot.extras.utils import extras_features
 from netutils.config.compliance import feature_compliance
+from xmldiff import main, actions
+
 
 from nautobot_golden_config.choices import ComplianceRuleConfigTypeChoice, ConfigPlanTypeChoice, RemediationTypeChoice
 from nautobot_golden_config.utilities.constant import ENABLE_SOTAGG, PLUGIN_CFG
@@ -118,6 +120,41 @@ def _normalize_diff(diff, path_to_diff):
     }
 
 
+def _get_xml_compliance(obj):
+    """This function performs the actual compliance for xml serializable data."""
+
+    def _normalize_diff(diff):
+        """Format the diff output to a list of nodes with values that have updated."""
+        formatted_diff = []
+        for operation in diff:
+            if isinstance(operation, actions.UpdateTextIn):
+                formatted_operation = f"{operation.node}, {operation.text}"
+                formatted_diff.append(formatted_operation)
+        return "\n".join(formatted_diff)
+
+    # Options for the diff operation. These are set to prefer updates over node insertions/deletions.
+    diff_options = {
+        "F": 0.1,
+        "fast_match": True,
+    }
+    missing = main.diff_texts(obj.actual, obj.intended, diff_options=diff_options)
+    extra = main.diff_texts(obj.intended, obj.actual, diff_options=diff_options)
+
+    compliance = not missing and not extra
+    compliance_int = int(compliance)
+    ordered = obj.ordered
+    missing = _null_to_empty(_normalize_diff(missing))
+    extra = _null_to_empty(_normalize_diff(extra))
+
+    return {
+        "compliance": compliance,
+        "compliance_int": compliance_int,
+        "ordered": ordered,
+        "missing": missing,
+        "extra": extra,
+    }
+
+
 def _verify_get_custom_compliance_data(compliance_details):
     """This function verifies the data is as expected when a custom function is used."""
     for val in ["compliance", "compliance_int", "ordered", "missing", "extra"]:
@@ -171,6 +208,7 @@ def _get_hierconfig_remediation(obj):
 FUNC_MAPPER = {
     ComplianceRuleConfigTypeChoice.TYPE_CLI: _get_cli_compliance,
     ComplianceRuleConfigTypeChoice.TYPE_JSON: _get_json_compliance,
+    ComplianceRuleConfigTypeChoice.TYPE_XML: _get_xml_compliance,
     RemediationTypeChoice.TYPE_HIERCONFIG: _get_hierconfig_remediation,
 }
 # The below conditionally add the custom provided compliance type
@@ -249,13 +287,13 @@ class ComplianceRule(PrimaryModel):  # pylint: disable=too-many-ancestors
     match_config = models.TextField(
         blank=True,
         verbose_name="Config to Match",
-        help_text="The config to match that is matched based on the parent most configuration. E.g.: For CLI `router bgp` or `ntp`. For JSON this is a top level key name.",
+        help_text="The config to match that is matched based on the parent most configuration. E.g.: For CLI `router bgp` or `ntp`. For JSON this is a top level key name. For XML this is a xpath query.",
     )
     config_type = models.CharField(
         max_length=20,
         default=ComplianceRuleConfigTypeChoice.TYPE_CLI,
         choices=ComplianceRuleConfigTypeChoice,
-        help_text="Whether the configuration is in CLI or JSON/structured format.",
+        help_text="Whether the configuration is in CLI, JSON, or XML format.",
     )
     custom_compliance = models.BooleanField(
         default=False, help_text="Whether this Compliance Rule is proceeded as custom."

nautobot_golden_config/nornir_plays/config_compliance.py

@@ -6,8 +6,9 @@
 import os
 from collections import defaultdict
 from datetime import datetime
-
 from django.utils.timezone import make_aware
+from lxml import etree  # nosec
+
 from nautobot_plugin_nornir.constants import NORNIR_SETTINGS
 from nautobot_plugin_nornir.plugins.inventory.nautobot_orm import NautobotORMInventory
 from netutils.config.compliance import _open_file_config, parser_map, section_config
@@ -20,8 +21,14 @@
 from nautobot_golden_config.models import ComplianceRule, ConfigCompliance, GoldenConfig
 from nautobot_golden_config.nornir_plays.processor import ProcessGoldenConfig
 from nautobot_golden_config.utilities.db_management import close_threaded_db_connections
-from nautobot_golden_config.utilities.helper import get_json_config, render_jinja_template, verify_settings
 from nautobot_golden_config.utilities.logger import NornirLogger
+from nautobot_golden_config.utilities.helper import (
+    get_json_config,
+    get_xml_config,
+    render_jinja_template,
+    verify_settings,
+    get_xml_subtree_with_full_path,
+)
 
 InventoryPluginRegister.register("nautobot-inventory", NautobotORMInventory)
 LOGGER = logging.getLogger(__name__)
@@ -64,6 +71,24 @@ def get_config_element(rule, config, obj, logger):
         else:
             config_element = config_json
 
+    elif rule["obj"].config_type == ComplianceRuleConfigTypeChoice.TYPE_XML:
+        config_xml = get_xml_config(config)
+
+        if not config_xml:
+            error_msg = "`E3002:` Unable to interpret configuration as XML."
+            logger.error(error_msg, extra={"object": obj})
+            raise NornirNautobotException(error_msg)
+
+        if rule["obj"].match_config:
+            try:
+                config_element = get_xml_subtree_with_full_path(config_xml, rule["obj"].match_config)
+            except etree.XPathError as err:
+                error_msg = f"`E3031:` Invalid XPath expression - `{rule['obj'].match_config}`"
+                logger.error(error_msg, extra={"object": obj})
+                raise NornirNautobotException(error_msg) from err
+        else:
+            config_element = etree.tostring(config_xml, encoding="unicode", pretty_print=True)
+
     elif rule["obj"].config_type == ComplianceRuleConfigTypeChoice.TYPE_CLI:
         if obj.platform.network_driver_mappings["netutils_parser"] not in parser_map:
             error_msg = f"`E3003:` There is currently no CLI-config parser support for platform network_driver `{obj.platform.network_driver}`, preemptively failed."

nautobot_golden_config/template_content.py

@@ -12,18 +12,19 @@ class ConfigComplianceDeviceCheck(PluginTemplateExtension):  # pylint: disable=a
 
     model = "dcim.device"
 
-    def get_device(self):
-        """Get device object."""
+    @property
+    def device(self):
+        """Device presented in detail view."""
         return self.context["object"]
 
     def right_page(self):
         """Content to add to the configuration compliance."""
-        comp_obj = ConfigCompliance.objects.filter(device=self.get_device()).values("rule__feature__name", "compliance")
+        comp_obj = ConfigCompliance.objects.filter(device=self.device).values("rule__feature__name", "compliance")
         if not comp_obj:
             return ""
         extra_context = {
             "compliance": comp_obj,
-            "device": self.get_device(),
+            "device": self.device,
             "template_type": "devicetab",
         }
         return self.render(
@@ -34,18 +35,21 @@ def right_page(self):
     def detail_tabs(self):
         """Add a Configuration Compliance tab to the Device detail view if the Configuration Compliance associated to it."""
         try:
-            return [
-                {
-                    "title": "Configuration Compliance",
-                    "url": reverse(
-                        "plugins:nautobot_golden_config:configcompliance_devicetab",
-                        kwargs={"pk": self.get_device().pk},
-                    ),
-                }
-            ]
+            if ConfigCompliance.objects.filter(device=self.device):
+                return [
+                    {
+                        "title": "Configuration Compliance",
+                        "url": reverse(
+                            "plugins:nautobot_golden_config:configcompliance_devicetab",
+                            kwargs={"pk": self.device.pk},
+                        ),
+                    }
+                ]
         except ObjectDoesNotExist:
             return []
 
+        return []
+
 
 class ConfigComplianceLocationCheck(PluginTemplateExtension):  # pylint: disable=abstract-method
     """App extension class for config compliance."""

nautobot_golden_config/templates/nautobot_golden_config/configcompliance_devicetab.html

@@ -111,7 +111,13 @@
             <tr>
                 <td style="width:250px">Configuration</td>
                 <td class="config_hover">
-                    <span id="{{ item.rule|slugify }}_actual"><pre>{{ item.actual|placeholder|condition_render_json }}</pre></span>
+                    {% if item.rule.config_type == "xml" %}
+                        <span id="{{ item.rule|slugify }}_actual"><pre><code class="language-xml">{{ item.actual|placeholder }}</code></pre></span>
+                    {% elif item.rule.config_type == "json" %}
+                        <span id="{{ item.rule|slugify }}_actual"><pre><code class="language-json">{{ item.actual|placeholder|condition_render_json }}</code></pre></span>
+                    {% else %}
+                        <span id="{{ item.rule|slugify }}_actual"><pre>{{ item.actual|placeholder }}</pre></span>
+                    {% endif %}
                     <span class="config_hover_button">
                         <button class="btn btn-inline btn-default hover_copy_button" data-clipboard-target="#{{ item.rule|slugify }}_actual">
                             <span class="mdi mdi-content-copy"></span>
@@ -123,7 +129,13 @@
             <tr>
                 <td style="width:250px">Intended Configuration</td>
                 <td class="config_hover">
-                    <span id="{{ item.rule|slugify }}_intended"><pre>{{ item.intended|placeholder|condition_render_json }}</pre></span>
+                    {% if item.rule.config_type == "xml" %}
+                        <span id="{{ item.rule|slugify }}_intended"><pre><code class="language-xml">{{ item.intended|placeholder }}</code></pre></span>
+                    {% elif item.rule.config_type == "json" %}
+                        <span id="{{ item.rule|slugify }}_intended"><pre><code class="language-json">{{ item.intended|placeholder|condition_render_json }}</code></pre></span>
+                    {% else %}
+                        <span id="{{ item.rule|slugify }}_intended"><pre>{{ item.intended|placeholder }}</pre></span>
+                    {% endif %}
                     <span class="config_hover_button">
                         <button class="btn btn-inline btn-default hover_copy_button" data-clipboard-target="#{{ item.rule|slugify }}_intended">
                             <span class="mdi mdi-content-copy"></span>
@@ -134,7 +146,13 @@
             <tr>
                 <td style="width:250px">Actual Configuration</td>
                 <td class="config_hover">
-                    <span id="{{ item.rule|slugify }}_actual"><pre>{{ item.actual|placeholder|condition_render_json }}</pre></span>
+                    {% if item.rule.config_type == "xml" %}
+                        <span id="{{ item.rule|slugify }}_actual"><pre><code class="language-xml">{{ item.actual|placeholder }}</code></pre></span>
+                    {% elif item.rule.config_type == "json" %}
+                        <span id="{{ item.rule|slugify }}_actual"><pre><code class="language-json">{{ item.actual|placeholder|condition_render_json }}</code></pre></span>
+                    {% else %}
+                        <span id="{{ item.rule|slugify }}_actual"><pre>{{ item.actual|placeholder }}</pre></span>
+                    {% endif %}
                     <span class="config_hover_button">
                         <button class="btn btn-inline btn-default hover_copy_button" data-clipboard-target="#{{ item.rule|slugify }}_actual">
                             <span class="mdi mdi-content-copy"></span>

nautobot_golden_config/tests/conftest.py

@@ -284,6 +284,33 @@ def create_feature_rule_cli_with_remediation(device, feature="foo3", rule="cli")
     return rule
 
 
+def create_feature_rule_xml(device, feature="foo4", rule="xml"):
+    """Creates a Feature/Rule Mapping and Returns the rule."""
+    feature_obj, _ = ComplianceFeature.objects.get_or_create(slug=feature, name=feature)
+    rule = ComplianceRule(
+        feature=feature_obj,
+        platform=device.platform,
+        config_type=ComplianceRuleConfigTypeChoice.TYPE_XML,
+        config_ordered=False,
+    )
+    rule.save()
+    return rule
+
+
+def create_feature_rule_xml_with_remediation(device, feature="foo5", rule="xml"):
+    """Creates a Feature/Rule Mapping with remediation enabled and Returns the rule."""
+    feature_obj, _ = ComplianceFeature.objects.get_or_create(slug=feature, name=feature)
+    rule = ComplianceRule(
+        feature=feature_obj,
+        platform=device.platform,
+        config_type=ComplianceRuleConfigTypeChoice.TYPE_XML,
+        config_ordered=False,
+        config_remediation=True,
+    )
+    rule.save()
+    return rule
+
+
 def create_feature_rule_cli(device, feature="foo_cli"):
     """Creates a Feature/Rule Mapping and Returns the rule."""
     feature_obj, _ = ComplianceFeature.objects.get_or_create(slug=feature, name=feature)