Merge Develop back into Next

.flake8

@@ -1,4 +1,4 @@
 [flake8]
 # E501: Line length is enforced by Black, so flake8 doesn't need to check it
 # W503: Black disagrees with this rule, as does PEP 8; Black wins
-ignore = E501, W503, F811, F401, F405, E203
+ignore = E501, W503

development/development.env

@@ -48,4 +48,6 @@ ENABLE_INTENDED=True
 ENABLE_BACKUP=True
 ENABLE_SOTAGG=True
 ENABLE_POSTPROCESSING=True
+ENABLE_PLAN=True
+ENABLE_DEPLOY=True
 ALLOWED_OS=all

development/nautobot_config.py

@@ -4,11 +4,9 @@
 import sys
 
 from django.utils.module_loading import import_string
-
 from nautobot.core.settings import *  # noqa: F403
 from nautobot.core.settings_funcs import is_truthy, parse_redis_connection
 
-
 #
 # Misc. settings
 #
@@ -172,8 +170,10 @@
         "enable_compliance": is_truthy(os.environ.get("ENABLE_COMPLIANCE", True)),
         "enable_intended": is_truthy(os.environ.get("ENABLE_INTENDED", True)),
         "enable_sotagg": is_truthy(os.environ.get("ENABLE_SOTAGG", True)),
-        "sot_agg_transposer": os.environ.get("SOT_AGG_TRANSPOSER"),
         "enable_postprocessing": is_truthy(os.environ.get("ENABLE_POSTPROCESSING", True)),
+        "enable_plan": is_truthy(os.environ.get("ENABLE_PLAN", True)),
+        "enable_deploy": is_truthy(os.environ.get("ENABLE_DEPLOY", True)),
+        "sot_agg_transposer": os.environ.get("SOT_AGG_TRANSPOSER"),
         "postprocessing_callables": os.environ.get("POSTPROCESSING_CALLABLES", []),
         "postprocessing_subscribed": os.environ.get("POSTPROCESSING_SUBSCRIBED", []),
         "jinja_env": {
@@ -198,7 +198,7 @@
 
 # Modify django_jinja Environment for test cases
 django_jinja_config = None
-for template in TEMPLATES:
+for template in TEMPLATES:  # noqa: F405
     if template["BACKEND"].startswith("django_jinja"):
         django_jinja_config = template
 
@@ -211,4 +211,4 @@
     jinja_options["undefined"] = "jinja2.StrictUndefined"
 
 # Import filter function to have it register filter with django_jinja
-from nautobot_golden_config.tests import jinja_filters  # noqa: E402
+from nautobot_golden_config.tests import jinja_filters  # noqa: E402, F401

docs/admin/admin_install.md

@@ -54,8 +54,10 @@ PLUGINS_CONFIG = {
         "enable_compliance": True,
         "enable_intended": True,
         "enable_sotagg": True,
-        "sot_agg_transposer": None,
+        "enable_plan": True,
+        "enable_deploy": True,
         "enable_postprocessing": False,
+        "sot_agg_transposer": None,
         "postprocessing_callables": [],
         "postprocessing_subscribed": [],
         "platform_slug_map": None,
@@ -90,14 +92,16 @@ sudo systemctl restart nautobot nautobot-worker nautobot-scheduler
 The plugin behavior can be controlled with the following list of settings.
 
 !!! note
-    The `enable_backup`, `enable_compliance`, `enable_intended`, `enable_sotagg` and `enable_postprocessing` will toggle inclusion of the entire component.
+    The `enable_backup`, `enable_compliance`, `enable_intended`, `enable_sotagg`, `enable_plan`, `enable_deploy`, and `enable_postprocessing` will toggle inclusion of the entire component.
 
 | Key                       | Example                       | Default | Description                                                                                                                                                                |
 | ------------------------- | ----------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | enable_backup             | True                          | True    | A boolean to represent whether or not to run backup configurations within the plugin.                                                                                      |
 | enable_compliance         | True                          | True    | A boolean to represent whether or not to run the compliance process within the plugin.                                                                                     |
 | enable_intended           | True                          | True    | A boolean to represent whether or not to generate intended configurations within the plugin.                                                                               |
 | enable_sotagg             | True                          | True    | A boolean to represent whether or not to provide a GraphQL query per device to allow the intended configuration to provide data variables to the plugin.                   |
+| enable_plan               | True                          | True    | A boolean to represent whether or not to allow the config plan job to run.                                                                                                          |
+| enable_deploy             | True                          | True    | A boolean to represent whether or not to be able to deploy configs to network devices.                                                                             |
 | enable_postprocessing     | True                          | False    | A boolean to represent whether or not to generate intended configurations to push, with extra processing such as secrets rendering.                                       |
 | postprocessing_callables  | ['mypackage.myfunction']      | []      | A list of function paths, in dotted format, that are appended to the available methods for post-processing the intended configuration, for instance, the `render_secrets`. |
 | postprocessing_subscribed | ['mypackage.myfunction']      | []      | A list of function paths, that should exist as postprocessing_callables, that defines the order of application of during the post-processing process.                      |

docs/admin/compatibility_matrix.md

@@ -16,3 +16,4 @@ While that last supported version will not be strictly enforced via the `max_ver
 | 1.3.X                 | 1.4.0                          | 1.5.2 [Official]              |
 | 1.4.X                 | 1.5.3                          | 1.5.99 [Official]             |
 | 1.5.X                 | 1.6.1                          | 1.6.99 [Official]             |
+| 1.6.X                 | 1.6.1                          | 1.6.99 [Official]             |

docs/admin/release_notes/version_1.6.md

@@ -0,0 +1,21 @@
+# v1.6 Release Notes
+
+- Add ability to generate ConfigPlans for configurations that need to be deployed, based on multiple plan types.
+- Add a job that can deploy config_set based on a generated ConfigPlan object.
+- Add functionality to compliance result to provide a Remediation plan.
+- Supports Nautobot >=1.6.1,<2.0.0.
+
+## v1.6.0 - 2023-09
+
+### Added
+
+- [#573](https://github.com/nautobot/nautobot-plugin-golden-config/pull/573) - Added the ability to generate remediation configurations and store in ConfigRemediation model
+- [#573](https://github.com/nautobot/nautobot-plugin-golden-config/pull/573) - Added the ability to generate configurations that you plan to deploy from a variety of methods, such as Remediation, intended, manual, etc. via the ConfigPlan model.
+- [#573](https://github.com/nautobot/nautobot-plugin-golden-config/pull/573) - Added the ability to Deploy configurations from the ConfigPlan configurations to your network devices.
+
+### Fixed
+
+- [#585](https://github.com/nautobot/nautobot-plugin-golden-config/pull/585) - Remove Jquery dependency from Google APIs, inherit from Nautobot core instead.
+- [#577](https://github.com/nautobot/nautobot-plugin-golden-config/pull/577) - Fixed various forms fields and filters fields.
+- [#577](https://github.com/nautobot/nautobot-plugin-golden-config/pull/577) - Updated default has_sensitive_data boolean to False.
+- [#577](https://github.com/nautobot/nautobot-plugin-golden-config/pull/577) - Added warning message on views when required jobs are not enabled.

docs/dev/dev_adr.md

@@ -143,3 +143,17 @@ This function performs an additional permission validation, to check if the requ
 Over time device(s) platform may change; whether this is a device refresh or full replacement. A Django `post_save` signal is used on the `ConfigCompliance` model and provides a reliable and efficient way to manage configuration compliance objects. This signal deletes any `ConfigCompliance` objects that don't match the current platform. This decision was made to avoid compliance reporting inconsistencies that can arise when outdated or irrelevant objects remain in the database which were generated with the previous platform.
 
 This has a computational impact when updating a Device object's platform. This is similar to the computational impact of an SQL `cascade` option on a delete. This is largely unavoidable and should be limited in impact, such that it will only be the removal of the number of `ConfigCompliance` objects, which is no bigger than the number of  `Config Features`, which is generally intended to be a small amount.
+
+### Configuration Deployment and Remediation
+
+Configuration remediation and deployments of any of the attributes based on the configuration compliance object are calculated based on the last run of the `ConfigCompliance` job. After a configuration deployment to fix any of these attributes (remediation, intended, missing) a new `ConfigCompliance` job must be run before all the compliance results will be updated.
+
+
+### Manual ConfigPlans
+
+When generating a manual `ConfigPlan` the Jinja2 template render has access to Django ORM methods like `.all()`, this also means that methods like `.delete()` can be called, the `render_template` functionality used by Golden Config inherits a Jinja2 Sandbox exception that will block unsafe calls. Golden Config will simply re-raise the exception `jinja2.exceptions.SecurityError: > is not safely callable`.
+
+
+### Hidden Jobs and JobButtons
+
+The configuration deployment and plans features of Golden Config come packaged with Jobs and JobButtons to execute the functionality. In order to to provide a repeatable and consistent behavior these Jobs and JobButtons are designed to only be executed via specialized views. They're not intended to be executed manually from the Jobs/JobButtons menus.

docs/user/app_getting_started.md

@@ -4,6 +4,9 @@
 - [Backup Configuration](#backup-configuration)
 - [Intended Configuration](#intended-configuration)
 - [Compliance](#compliance)
+- [Config Remediation](#config-remediation)
+- [Config Plans](#config-plans)
+- [Config Deploy](#config-deploy)
 - [Load Properties from Git](#load-properties-from-git)
 
 # Backup Configuration
@@ -122,6 +125,53 @@ Compliance requires Backups and Intended Configurations in order to be executed.
 
 > For in-depth details see [Navigating Compliance](./app_feature_compliance.md)
 
+# Config Remediation
+
+Follow the steps below to get up and running for the configuration remediation element of the plugin.
+
+1. Navigate to `Golden Config -> Compliance Rules`.
+2. Select the rules in which you'd like to enable remediation on.
+3. Edit the `Compliance Rule` and turn on the `Remediation` toggle button.
+4. Run the `Compliance` job again which will generate the initial remediation plan for the feature.
+5. Navigate to `Golden Config -> Config Compliance`, select the device and notice a remediation section is not present for the compliance details for the feature.
+
+> For in-depth details see [Navigating Config Plans](./app_feature_remediation.md)
+
+# Config Plans
+
+Follow the steps below to get up and running for the configuration plans element of the plugin.
+
+1. Enable the feature in the `PLUGIN_SETTINGS`. The configuration should have `"enable_plan": True` set in the `PLUGINS_CONFIG` dictionary for `nautobot_golden_config`.
+2. Follow the steps in [Compliance](#compliance).
+    - Compliance is necessary if ConfigPlans will be generated utilizing any of the attributes provided by a Compliance object.
+    - This step may be skipped if only `manual` ConfigPlans are going to be generated.
+3. Create a ConfigPlan
+
+    1. Navigate to `Golden Config -> Config Plans`
+    2. Click on `ADD` button.
+    3. Fill out the plan details and plan filters.
+        - The options dynamically change in the form based on the `plan type` selected.
+        - If the `plan type` is Intended, Remediation, Missing.
+            - Select the Compliance Features to use to generate the plan.  If none are selected all features will be in scope.
+        - If the `plan type` is Manual.
+            - Create a manual plan to accomplish the goal. Note: Access to `obj` is available to dynamically populate fields via Jinja2 syntax.
+    4. Click `Generate`
+
+> For in-depth details see [Navigating Config Plans](./app_feature_config_plans.md)
+
+# Config Deploy
+
+Follow the steps below to get up and running for the configuration deployment element of the plugin.
+
+1. Enable the feature in the `PLUGIN_SETTINGS`. The configuration should have `"enable_deploy": True` set in the `PLUGINS_CONFIG` dictionary for `nautobot_golden_config`.
+2. Follow the steps in [Config Plans](#config-plans).
+3. Navigate to the specific ConfigPlan to deploy, or multi-select them from the ConfigPlan list view.
+    - If deploying from a specific ConfigPlan object. Click `Deploy` button and accept the warnings.
+    - If deploying from the ConfigPlan list view. Click `Deploy Selected` button and accept the warnings.
+4. Interpret the results from the popup modal and navigate to the job result as needed for more details.
+
+> Config Deployments utilize the dispatchers from nornir-nautobot just like the other functionality of Golden Config. See [Troubleshooting Dispatchers](./troubleshooting/troubleshoot_dispatchers.md) for more details.
+
 # Load Properties from Git
 
 Golden Config properties include: Compliance Features, Compliance Rules, Config Removals, and Config Replacements. They can be created via the UI, API, or alternatively you can load these properties from a Git repository, defined in YAML files following the this directory structure (you can skip any of them if not apply):

docs/user/troubleshooting/troubleshoot_general.md

@@ -64,3 +64,7 @@ If none of these troubleshooting steps helped identify the problem please visit
 
 - [Troubleshoot Credentials](./troubleshoot_credentials.md)
 - [Troubleshoot Dispatchers](./troubleshoot_dispatchers.md)
+
+### Detailed Diagrams
+
+Golden config flow diagrams are available on the repositories [wiki](https://github.com/nautobot/nautobot-plugin-golden-config/wiki#diagrams).

mkdocs.yml

@@ -120,6 +120,7 @@ nav:
       - Compatibility Matrix: "admin/compatibility_matrix.md"
       - Release Notes:
           - "admin/release_notes/index.md"
+          - v1.6: "admin/release_notes/version_1.6.md"
           - v1.5: "admin/release_notes/version_1.5.md"
           - v1.4: "admin/release_notes/version_1.4.md"
           - v1.3: "admin/release_notes/version_1.3.md"

nautobot_golden_config/__init__.py

@@ -19,7 +19,7 @@ class GoldenConfig(PluginConfig):
     version = __version__
     author = "Network to Code, LLC"
     author_email = "[email protected]"
-    description = "Nautobot Apps that embraces NetDevOps and automates configuration backups, performs configuration compliance, and generates intended configurations. Includes native Git integration and gives users the flexibility to mix and match the supported features."
+    description = "Nautobot Apps that embraces NetDevOps and automates configuration backups, performs configuration compliance, generates intended configurations, and has config remediation and deployment features. Includes native Git integration and gives users the flexibility to mix and match the supported features."
     base_url = "golden-config"
     min_version = "1.6.1"
     max_version = "1.99"
@@ -29,6 +29,8 @@ class GoldenConfig(PluginConfig):
         "enable_intended": True,
         "enable_sotagg": True,
         "enable_postprocessing": False,
+        "enable_plan": True,
+        "enable_deploy": True,
         "postprocessing_callables": [],
         "postprocessing_subscribed": [],
         "per_feature_bar_width": 0.3,

nautobot_golden_config/filters.py

@@ -7,13 +7,8 @@
 from nautobot.extras.filters import NautobotFilterSet, StatusFilter
 from nautobot.extras.models import JobResult, Status
 from nautobot.tenancy.models import Tenant, TenantGroup
-from nautobot.utilities.filters import (
-    BaseFilterSet,
-    MultiValueDateTimeFilter,
-    NameSlugSearchFilterSet,
-    TagFilter,
-    TreeNodeMultipleChoiceFilter,
-)
+from nautobot.utilities.filters import MultiValueDateTimeFilter, TagFilter, TreeNodeMultipleChoiceFilter
+
 from nautobot_golden_config import models
 
 
@@ -368,7 +363,7 @@ class Meta:
         fields = ["id", "name", "slug", "weight", "backup_repository", "intended_repository", "jinja_repository"]
 
 
-class RemediationSettingFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
+class RemediationSettingFilterSet(NautobotFilterSet):
     """Inherits Base Class CustomFieldModelFilterSet."""
 
     q = django_filters.CharFilter(
@@ -385,12 +380,6 @@ class RemediationSettingFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
         queryset=Platform.objects.all(),
         label="Platform ID",
     )
-    remediation_type = django_filters.ModelMultipleChoiceFilter(
-        field_name="remediation_type",
-        queryset=models.RemediationSetting.objects.all(),
-        to_field_name="remediation_type",
-        label="Remediation Type",
-    )
 
     def search(self, queryset, name, value):  # pylint: disable=unused-argument
         """Perform the filtered search."""
@@ -403,10 +392,10 @@ class Meta:
         """Boilerplate filter Meta data for Remediation Setting."""
 
         model = models.RemediationSetting
-        fields = ["id", "platform", "remediation_type"]
+        fields = ["id", "remediation_type"]
 
 
-class ConfigPlanFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
+class ConfigPlanFilterSet(NautobotFilterSet):
     """Inherits Base Class BaseFilterSet."""
 
     q = django_filters.CharFilter(
@@ -435,9 +424,13 @@ class ConfigPlanFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
         to_field_name="name",
         label="Feature Name",
     )
-    job_result_id = django_filters.ModelMultipleChoiceFilter(
+    plan_result_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=JobResult.objects.filter(config_plan__isnull=False).distinct(),
+        label="Plan JobResult ID",
+    )
+    deploy_result_id = django_filters.ModelMultipleChoiceFilter(
         queryset=JobResult.objects.filter(config_plan__isnull=False).distinct(),
-        label="JobResult ID",
+        label="Deploy JobResult ID",
     )
     change_control_id = django_filters.CharFilter(
         field_name="change_control_id",
@@ -466,4 +459,4 @@ class Meta:
         """Boilerplate filter Meta data for Config Plan."""
 
         model = models.ConfigPlan
-        fields = ["id", "device", "created", "plan_type", "feature", "change_control_id", "status"]
+        fields = ["id", "created", "change_control_id", "plan_type"]