Merge Next (1.5.0) into develop

.github/workflows/ci.yml

@@ -79,8 +79,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.8"]
-        nautobot-version: ["1.5.3"]
+        python-version: ["3.11"]
+        nautobot-version: ["1.6.1"]
     env:
       INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
@@ -116,20 +116,20 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         db-backend: ["postgresql"]
         nautobot-version: ["latest"]
         # The include is a method to limit the amount of jobs ran. This essentially
         # means that in addition to standard postgres and stable, also the lowest
         # supported version and with mysql
         include:
-          - python-version: "3.10"
+          - python-version: "3.11"
             db-backend: "postgresql"
-            nautobot-version: "1.5.3"
-          - python-version: "3.7"
+            nautobot-version: "1.6.1"
+          - python-version: "3.8"
             db-backend: "mysql"
-            nautobot-version: "1.5.3"
-          - python-version: "3.10"
+            nautobot-version: "1.6.1"
+          - python-version: "3.11"
             db-backend: "mysql"
             nautobot-version: "latest"
     runs-on: "ubuntu-20.04"
@@ -177,7 +177,7 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v2"
         with:
-          python-version: "3.9"
+          python-version: "3.11"
       - name: "Install Python Packages"
         run: "pip install poetry"
       - name: "Set env"
@@ -206,7 +206,7 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v2"
         with:
-          python-version: "3.9"
+          python-version: "3.11"
       - name: "Install Python Packages"
         run: "pip install poetry"
       - name: "Set env"

development/Dockerfile

@@ -6,11 +6,11 @@
 # -------------------------------------------------------------------------------------
 # !!! USE CAUTION WHEN MODIFYING LINES BELOW
 
-# Accepts a desired Nautobot version as build argument, default to 1.5.3
-ARG NAUTOBOT_VER="1.5.3"
+# Accepts a desired Nautobot version as build argument, default to 1.6.1
+ARG NAUTOBOT_VER="1.6.1"
 
-# Accepts a desired Python version as build argument, default to 3.7
-ARG PYTHON_VER="3.7"
+# Accepts a desired Python version as build argument, default to 3.11
+ARG PYTHON_VER="3.11"
 
 # Retrieve published development image of Nautobot base which should include most CI dependencies
 FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER}
@@ -26,7 +26,6 @@ ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT}
 # and CI and local development may have a newer version of Poetry
 # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary
 # We also don't need virtual environments in container
-ARG POETRY_VERSION=1.5.1
 RUN curl -sSL https://install.python-poetry.org -o /tmp/install-poetry.py && \
     python /tmp/install-poetry.py && \
     rm -f /tmp/install-poetry.py && \

development/nautobot_config.py

@@ -3,8 +3,10 @@
 import os
 import sys
 
+from django.utils.module_loading import import_string
+
 from nautobot.core.settings import *  # noqa: F403
-from nautobot.core.settings_funcs import parse_redis_connection
+from nautobot.core.settings_funcs import is_truthy, parse_redis_connection
 
 
 #
@@ -174,6 +176,11 @@
         "enable_postprocessing": is_truthy(os.environ.get("ENABLE_POSTPROCESSING", True)),
         "postprocessing_callables": os.environ.get("POSTPROCESSING_CALLABLES", []),
         "postprocessing_subscribed": os.environ.get("POSTPROCESSING_SUBSCRIBED", []),
+        "jinja_env": {
+            "undefined": import_string("jinja2.StrictUndefined"),
+            "trim_blocks": is_truthy(os.getenv("NAUTOBOT_JINJA_ENV_TRIM_BLOCKS", True)),
+            "lstrip_blocks": is_truthy(os.getenv("NAUTOBOT_JINJA_ENV_LSTRIP_BLOCKS", False)),
+        },
         # The platform_slug_map maps an arbitrary platform slug to its corresponding parser.
         # Use this if the platform slug names in your Nautobot instance don't correspond exactly
         # to the Nornir driver names ("arista_eos", "cisco_ios", etc.).

docs/admin/admin_install.md

@@ -59,6 +59,11 @@ PLUGINS_CONFIG = {
         "postprocessing_callables": [],
         "postprocessing_subscribed": [],
         "platform_slug_map": None,
+        "jinja_env": {
+            "undefined": StrictUndefined,  # jinja2.StrictUndefined
+            "trim_blocks": True,
+            "lstrip_blocks": False,
+        },
         # "get_custom_compliance": "my.custom_compliance.func"
     },
 }
@@ -93,17 +98,29 @@ The plugin behavior can be controlled with the following list of settings.
 | enable_compliance         | True                          | True    | A boolean to represent whether or not to run the compliance process within the plugin.                                                                                     |
 | enable_intended           | True                          | True    | A boolean to represent whether or not to generate intended configurations within the plugin.                                                                               |
 | enable_sotagg             | True                          | True    | A boolean to represent whether or not to provide a GraphQL query per device to allow the intended configuration to provide data variables to the plugin.                   |
-| enable_postprocessing     | True                          | False    | A boolean to represent whether or not to generate intended configurations to push, with extra processing such as secrets rendering.                                        |
+| enable_postprocessing     | True                          | False    | A boolean to represent whether or not to generate intended configurations to push, with extra processing such as secrets rendering.                                       |
 | postprocessing_callables  | ['mypackage.myfunction']      | []      | A list of function paths, in dotted format, that are appended to the available methods for post-processing the intended configuration, for instance, the `render_secrets`. |
 | postprocessing_subscribed | ['mypackage.myfunction']      | []      | A list of function paths, that should exist as postprocessing_callables, that defines the order of application of during the post-processing process.                      |
-| platform_slug_map         | {"cisco_wlc": "cisco_aireos"} | None    | A dictionary in which the key is the platform slug and the value is what netutils uses in any "network_os" parameter within `netutils.config.compliance.parser_map`.                                                      |
+| platform_slug_map         | {"cisco_wlc": "cisco_aireos"} | None    | A dictionary in which the key is the platform slug and the value is what netutils uses in any "network_os" parameter within `netutils.config.compliance.parser_map`.       |
 | sot_agg_transposer        | "mypkg.transposer"            | None    | A string representation of a function that can post-process the graphQL data.                                                                                              |
 | per_feature_bar_width     | 0.15                          | 0.15    | The width of the table bar within the overview report                                                                                                                      |
 | per_feature_width         | 13                            | 13      | The width in inches that the overview table can be.                                                                                                                        |
 | per_feature_height        | 4                             | 4       | The height in inches that the overview table can be.                                                                                                                       |
+| jinja_env | {"lstrip_blocks": False} | See Note Below | A dictionary of Jinja2 Environment options compatible with Jinja2.SandboxEnvironment() |
 
 !!! note
     Over time the compliance report will become more dynamic, but for now allow users to configure the `per_*` configs in a way that fits best for them.
 
 !!! note
     Review [`nautobot_plugin_nornir`](https://docs.nautobot.com/projects/plugin-nornir/en/latest/user/app_feature_dispatcher/) for Nornir and dispatcher configuration options.
+
+!!! note
+    Defaults for Jinja2 environment settings (`jinja_env`) are as follows:
+
+    ```python
+        jinja_env = {
+            "undefined": import_string("jinja2.StrictUndefined"),
+            "trim_blocks": True,
+            "lstrip_blocks": False,
+        }
+    ```

docs/admin/compatibility_matrix.md

@@ -15,3 +15,4 @@ While that last supported version will not be strictly enforced via the `max_ver
 | 1.2.X                 | 1.4.0                          | 1.5.2 [Official]              |
 | 1.3.X                 | 1.4.0                          | 1.5.2 [Official]              |
 | 1.4.X                 | 1.5.3                          | 1.5.99 [Official]             |
+| 1.5.X                 | 1.6.1                          | 1.6.99 [Official]             |

docs/admin/release_notes/version_1.5.md

@@ -0,0 +1,27 @@
+# v1.5 Release Notes
+
+- Add ability to update JSON based compliance via a job and have git integrations.
+- Made custom compliance a boolean to support both JSON or CLI custom compliance types.
+- Add metrics for Golden Config plugin.
+- Add jinja settings support.
+- Updated Filters for various models, including adding an experimental `_isnull` on DateTime objects.
+- Supports Nautobot >=1.6.1,<2.0.0.
+
+## v1.5.0 - 2023-08
+
+### Added
+
+- [455](https://github.com/nautobot/nautobot-plugin-golden-config/pull/455) - Add metrics for Golden Config plugin.
+- [485](https://github.com/nautobot/nautobot-plugin-golden-config/pull/485) - Custom compliance for CLI and JSON rules.
+- [487](https://github.com/nautobot/nautobot-plugin-golden-config/pull/487) - Implement native JSON support.
+- [527](https://github.com/nautobot/nautobot-plugin-golden-config/pull/527) - Add the ability to update Jinja environment setting from nautobot_config.
+- [558](https://github.com/nautobot/nautobot-plugin-golden-config/pull/558) - Updated Filters for various models, including adding an experimental `_isnull` on DateTime objects.
+
+### Changed
+
+- [485](https://github.com/nautobot/nautobot-plugin-golden-config/pull/485) - Changed the behavior of custom compliance to a boolean vs toggle between cli, json, and custom.
+
+### Fixed
+
+- [505](https://github.com/nautobot/nautobot-plugin-golden-config/pull/505) - fixes imports and choice definitions in the compliance nornir play.
+- [513](https://github.com/nautobot/nautobot-plugin-golden-config/pull/513) - Fixed issue with native JSON support with `get_config_element` function.

docs/dev/dev_environment.md

@@ -106,7 +106,7 @@ Each command can be executed with `invoke <command>`. All commands support the a
 
 #### Local Development Environment
 
-```
+```shell
   build            Build all docker images.
   debug            Start Nautobot and its dependencies in debug mode.
   destroy          Destroy all containers and volumes.
@@ -117,7 +117,7 @@ Each command can be executed with `invoke <command>`. All commands support the a
 
 #### Utility
 
-```
+```shell
   cli              Launch a bash shell inside the running Nautobot container.
   create-user      Create a new user in django (default: admin), will prompt for password.
   makemigrations   Run Make Migration in Django.
@@ -126,7 +126,7 @@ Each command can be executed with `invoke <command>`. All commands support the a
 
 #### Testing
 
-```
+```shell
   bandit           Run bandit to validate basic static code security analysis.
   black            Run black to check that Python files adhere to its style standards.
   flake8           Run flake8 to check that Python files adhere to its style standards.
@@ -155,7 +155,7 @@ Poetry is used in lieu of the "virtualenv" commands and is leveraged in both env
 The `pyproject.toml` file outlines all of the relevant dependencies for the project:
 
 - `tool.poetry.dependencies` - the main list of dependencies.
-- `tool.poetry.dev-dependencies` - development dependencies, to facilitate linting, testing, and documentation building.
+- `tool.poetry.group.dev.dependencies` - development dependencies, to facilitate linting, testing, and documentation building.
 
 The `poetry shell` command is used to create and enable a virtual environment managed by Poetry, so all commands ran going forward are executed within the virtual environment. This is similar to running the `source venv/bin/activate` command with virtualenvs. To install project dependencies in the virtual environment, you should run `poetry install` - this will install **both** project and development dependencies.
 

docs/requirements.txt

@@ -1,6 +1,5 @@
-griffe==0.30.1
-mkdocs==1.3.1
-mkdocs-material==8.4.2
+mkdocs==1.5.2
+mkdocs-material==9.2.4
 mkdocs-version-annotations==1.0.0
 mkdocstrings==0.22.0
-mkdocstrings-python==1.1.2
+mkdocstrings-python==1.5.2

docs/user/app_faq.md

@@ -136,10 +136,14 @@ ERROR: 1
 
 If you receive this error, the issue is the secret key has been changed, and **does not** have anything to do with the Golden Config plugin. You can either delete the entries from your data source and the reference to those in the Golden Config settings or revert the secret key back so it matches the original deployment. Any issues opened will be closed and this faq referred to. If you still need help, feel free to join the Slack community.
 
-_I got a `preemptively failed` error, but I know my system is setup correctly?_
+## _I got a `preemptively failed` error, but I know my system is setup correctly?_
 
 These errors have been accurate so far, that is not to say that there is no way they could be a bug, but most commonly they have worked as expected thus far. Common issues include.
 
 * Incorrectly configured Secrets
 * Filtering to nothing when presumption is the filter works a certain way
 * Referencing an OS that is not recognized
+
+## _Why is the `_isnull` on DateTime filters considered experimental?_
+
+There are various ways we can create a programmatic interface, which may change the behavior or name, for now it should be considered experimental as we may update this strategy.

docs/user/app_feature_compliance.md

@@ -54,8 +54,12 @@ a database perspective, the job will never run successfully, rendering the addit
 The Feature is a unique identifier, that should prefer shorter names, as this effects the width of the compliance overview and thus it's readability as a
 best practice.
 
-The "Configs to Match" section represents the configuration root elements. This would be the parent most key only. Additionally, the match is based on
-what a line starts with only. Meaning, there is an implicit greediness to the matching. All matches must start form the beginning of the line.
+The "Config to Match" section represents the configuration root elements. This would be the parent most key only. Additionally, the match is based on
+"Config Type", which could be JSON or CLI. For CLI based configs, the match is based on what a line starts with only. Meaning, there is an implicit greediness to the matching. All matches must start form the beginning of the line.
+For JSON based configs, the match is based on JSON's structure top level key name.
+
+!!! note
+    "Config to Match" is mandatory for CLI configurations. If config to match is not defined for JSON, the complete JSON configuration will be compared. If the config to match is defined, comparison will take place only for defined keys.
 
 !!! note
     If the data is accidentally "corrupted" with a bad tested match, simply delete the devices an re-run the compliance process.

docs/user/app_feature_compliancecli.md

@@ -1,7 +1,7 @@
 # Configuration Compliance
 
 !!! note
-    This document provides instructions based on `CLI` based compliance.  The other options are `JSON` [structured data compliance](./app_feature_compliancejson.md) and [custom compliance](./app_feature_compliancecustom.md).
+    This document provides instructions for `CLI` configuration type based compliance.  The other option is `JSON` based [structured data compliance](./app_feature_compliancejson.md).
 
 ## Configuration Compliance Parsing Engine
 
@@ -14,13 +14,16 @@ configuration. The compliance engine has several features to better build work t
 3. The ability to dynamically understand parent/child relationships within the configurations.
 
 In regards to `1`, consider the following example of how to obtain service configurations:
+
 ```
 service
 no service
 ```
+
 Specific configurations that start with either of these commands can be grouped together.
 
 In regards to `2`, consider the configurations of SNMP on a nexus switch.
+
 ```
 snmp-server community secure group network-admin
 snmp-server community networktocode group network-operator
@@ -34,14 +37,17 @@ snmp-server community secure group network-admin
 ```
 
 In regards to `3`, consider the following example of BGP configuration.
+
 ```
 router bgp
 prefix-list
 ```
+
 All configurations that are a parent and child relationships would be considered within the parsing engine. Additionally, if one configuration line was
 wrong, only that line and the parents would be shown, not all lines or only the missing configuration without context of the parents, e.g. Given:
 
 Actual
+
 ```
 router bgp 65250
   router-id 10.0.10.5
@@ -54,6 +60,7 @@ router bgp 65250
 ```
 
 Intended
+
 ```
 router bgp 65250
   router-id 10.0.10.6
@@ -64,6 +71,7 @@ router bgp 65250
     remote-as 65250
     address-family ipv4 unicast
 ```
+
 Would result in the identifying the missing configurations as:
 
 ```

docs/user/app_feature_compliancecustom.md

@@ -1,12 +1,17 @@
 # Navigating Compliance With Custom Logic
 
+## Introduction
+Custom compliance is a powerful method to alter the default behaviour and results of the natively provided `CLI` and `JSON` config type based configuration compliance.
+The high level idea is to provide a Python code containing custom logic defined by the User. This code is called by the Golden Configuration Plugin in the compliance process,
+allowing the user to change the default behaviour of the Golden Configuration Plugin.
+
+
+
 ## Caveats
 
-- The compliance `rule` must be unique for the Custom `config-type`.
 - The data provided can come from either setting via the API like JSON or via match_config like CLI. It is up to the operator to enforce.
 - Does not make any accommodations for adding to git.
 - Mixing/Matching string (or CLI type) and JSON type compliance rules is **NOT** supported. A device should only have compliance rules from one or the other, and it is up to the operator to enforce.
-    - Applying a `match_config` presumes it is CLI type and not having one presumes it is JSON type.
 - If the developer of the `get_custom_compliance` is not cognizant, the UI experience of the details may not always be obvious what the issues are.
     - As an example, if the developer simply returns a `True` or `False` into the missing or extra dictionary, it will not be obvious to the user.
 - The developer is responsible for ensuring the proper data structure is maintained for the given rule.
@@ -50,6 +55,8 @@ The function provided in string path format, must be installed in the same envir
 
 ## Configuration
 
+To enable the custom compliance, navigate to the respective `Compliance Rule` and enable option "Custom Compliance" on this rule.
+
 The path to the function is set in the `get_custom_compliance` configuration parameter. This is the string representation of the function and must be in
 Python importable into Nautobot and the workers. This is a callable function and not a class or other object type.