Merge pull request #596 from nautobot/develop

Merge 1.6.0 into main
nautobot · Sep 15, 2023 · 748f1b3 · 748f1b3
2 parents 4001c9f + 36d66a9
commit 748f1b3
Show file tree

Hide file tree

Showing 75 changed files with 5,815 additions and 2,503 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,4 +1,4 @@
 [flake8]
 # E501: Line length is enforced by Black, so flake8 doesn't need to check it
 # W503: Black disagrees with this rule, as does PEP 8; Black wins
-ignore = E501, W503, F811, F401, F405, E203
+ignore = E501, W503
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@ creds.env
 nautobot_golden_config/transposer.py
 docker-compose.override.yml
 packages/
+invoke.yml
 
 # Ansible Retry Files
 *.retry

diff --git a/development/Dockerfile b/development/Dockerfile
@@ -64,7 +64,7 @@ RUN pip show nautobot | grep "^Version: " | sed -e 's/Version: /nautobot==/' > c
 # We can't use the entire freeze as it takes forever to resolve with rigidly fixed non-direct dependencies,
 #   especially those that are only direct to Nautobot but the container included versions slightly mismatch
 RUN poetry export -f requirements.txt --without-hashes --output poetry_freeze_base.txt
-RUN poetry export -f requirements.txt --dev --without-hashes --output poetry_freeze_all.txt
+RUN poetry export -f requirements.txt --with dev --without-hashes --output poetry_freeze_all.txt
 RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_dev.txt
 
 # Install all local project as editable, constrained on Nautobot version, to get any additional

diff --git a/development/development.env b/development/development.env
@@ -48,4 +48,6 @@ ENABLE_INTENDED=True
 ENABLE_BACKUP=True
 ENABLE_SOTAGG=True
 ENABLE_POSTPROCESSING=True
+ENABLE_PLAN=True
+ENABLE_DEPLOY=True
 ALLOWED_OS=all
diff --git a/development/docker-compose.base.yml b/development/docker-compose.base.yml
@@ -21,8 +21,9 @@ services:
         condition: "service_started"
       db:
         condition: "service_healthy"
-    <<: *nautobot-build
-    <<: *nautobot-base
+    <<:
+      - *nautobot-build
+      - *nautobot-base
   worker:
     entrypoint:
       - "sh"

diff --git a/development/nautobot_config.py b/development/nautobot_config.py
@@ -4,11 +4,9 @@
 import sys
 
 from django.utils.module_loading import import_string
-
 from nautobot.core.settings import *  # noqa: F403
 from nautobot.core.settings_funcs import is_truthy, parse_redis_connection
 
-
 #
 # Misc. settings
 #
@@ -172,8 +170,10 @@
         "enable_compliance": is_truthy(os.environ.get("ENABLE_COMPLIANCE", True)),
         "enable_intended": is_truthy(os.environ.get("ENABLE_INTENDED", True)),
         "enable_sotagg": is_truthy(os.environ.get("ENABLE_SOTAGG", True)),
-        "sot_agg_transposer": os.environ.get("SOT_AGG_TRANSPOSER"),
         "enable_postprocessing": is_truthy(os.environ.get("ENABLE_POSTPROCESSING", True)),
+        "enable_plan": is_truthy(os.environ.get("ENABLE_PLAN", True)),
+        "enable_deploy": is_truthy(os.environ.get("ENABLE_DEPLOY", True)),
+        "sot_agg_transposer": os.environ.get("SOT_AGG_TRANSPOSER"),
         "postprocessing_callables": os.environ.get("POSTPROCESSING_CALLABLES", []),
         "postprocessing_subscribed": os.environ.get("POSTPROCESSING_SUBSCRIBED", []),
         "jinja_env": {
@@ -198,7 +198,7 @@
 
 # Modify django_jinja Environment for test cases
 django_jinja_config = None
-for template in TEMPLATES:
+for template in TEMPLATES:  # noqa: F405
     if template["BACKEND"].startswith("django_jinja"):
         django_jinja_config = template
 
@@ -211,4 +211,4 @@
     jinja_options["undefined"] = "jinja2.StrictUndefined"
 
 # Import filter function to have it register filter with django_jinja
-from nautobot_golden_config.tests import jinja_filters  # noqa: E402
+from nautobot_golden_config.tests import jinja_filters  # noqa: E402, F401
diff --git a/docs/admin/admin_install.md b/docs/admin/admin_install.md
@@ -54,8 +54,10 @@ PLUGINS_CONFIG = {
         "enable_compliance": True,
         "enable_intended": True,
         "enable_sotagg": True,
-        "sot_agg_transposer": None,
+        "enable_plan": True,
+        "enable_deploy": True,
         "enable_postprocessing": False,
+        "sot_agg_transposer": None,
         "postprocessing_callables": [],
         "postprocessing_subscribed": [],
         "platform_slug_map": None,
@@ -90,14 +92,16 @@ sudo systemctl restart nautobot nautobot-worker nautobot-scheduler
 The plugin behavior can be controlled with the following list of settings.
 
 !!! note
-    The `enable_backup`, `enable_compliance`, `enable_intended`, `enable_sotagg` and `enable_postprocessing` will toggle inclusion of the entire component.
+    The `enable_backup`, `enable_compliance`, `enable_intended`, `enable_sotagg`, `enable_plan`, `enable_deploy`, and `enable_postprocessing` will toggle inclusion of the entire component.
 
 | Key                       | Example                       | Default | Description                                                                                                                                                                |
 | ------------------------- | ----------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | enable_backup             | True                          | True    | A boolean to represent whether or not to run backup configurations within the plugin.                                                                                      |
 | enable_compliance         | True                          | True    | A boolean to represent whether or not to run the compliance process within the plugin.                                                                                     |
 | enable_intended           | True                          | True    | A boolean to represent whether or not to generate intended configurations within the plugin.                                                                               |
 | enable_sotagg             | True                          | True    | A boolean to represent whether or not to provide a GraphQL query per device to allow the intended configuration to provide data variables to the plugin.                   |
+| enable_plan               | True                          | True    | A boolean to represent whether or not to allow the config plan job to run.                                                                                                          |
+| enable_deploy             | True                          | True    | A boolean to represent whether or not to be able to deploy configs to network devices.                                                                             |
 | enable_postprocessing     | True                          | False    | A boolean to represent whether or not to generate intended configurations to push, with extra processing such as secrets rendering.                                       |
 | postprocessing_callables  | ['mypackage.myfunction']      | []      | A list of function paths, in dotted format, that are appended to the available methods for post-processing the intended configuration, for instance, the `render_secrets`. |
 | postprocessing_subscribed | ['mypackage.myfunction']      | []      | A list of function paths, that should exist as postprocessing_callables, that defines the order of application of during the post-processing process.                      |

diff --git a/docs/admin/compatibility_matrix.md b/docs/admin/compatibility_matrix.md
@@ -16,3 +16,4 @@ While that last supported version will not be strictly enforced via the `max_ver
 | 1.3.X                 | 1.4.0                          | 1.5.2 [Official]              |
 | 1.4.X                 | 1.5.3                          | 1.5.99 [Official]             |
 | 1.5.X                 | 1.6.1                          | 1.6.99 [Official]             |
+| 1.6.X                 | 1.6.1                          | 1.6.99 [Official]             |
diff --git a/docs/admin/release_notes/version_1.6.md b/docs/admin/release_notes/version_1.6.md
@@ -0,0 +1,21 @@
+# v1.6 Release Notes
+
+- Add ability to generate ConfigPlans for configurations that need to be deployed, based on multiple plan types.
+- Add a job that can deploy config_set based on a generated ConfigPlan object.
+- Add functionality to compliance result to provide a Remediation plan.
+- Supports Nautobot >=1.6.1,<2.0.0.
+
+## v1.6.0 - 2023-09
+
+### Added
+
+- [#573](https://github.com/nautobot/nautobot-plugin-golden-config/pull/573) - Added the ability to generate remediation configurations and store in ConfigRemediation model
+- [#573](https://github.com/nautobot/nautobot-plugin-golden-config/pull/573) - Added the ability to generate configurations that you plan to deploy from a variety of methods, such as Remediation, intended, manual, etc. via the ConfigPlan model.
+- [#573](https://github.com/nautobot/nautobot-plugin-golden-config/pull/573) - Added the ability to Deploy configurations from the ConfigPlan configurations to your network devices.
+
+### Fixed
+
+- [#585](https://github.com/nautobot/nautobot-plugin-golden-config/pull/585) - Remove Jquery dependency from Google APIs, inherit from Nautobot core instead.
+- [#577](https://github.com/nautobot/nautobot-plugin-golden-config/pull/577) - Fixed various forms fields and filters fields.
+- [#577](https://github.com/nautobot/nautobot-plugin-golden-config/pull/577) - Updated default has_sensitive_data boolean to False.
+- [#577](https://github.com/nautobot/nautobot-plugin-golden-config/pull/577) - Added warning message on views when required jobs are not enabled.
diff --git a/docs/dev/dev_adr.md b/docs/dev/dev_adr.md
@@ -143,3 +143,17 @@ This function performs an additional permission validation, to check if the requ
 Over time device(s) platform may change; whether this is a device refresh or full replacement. A Django `post_save` signal is used on the `ConfigCompliance` model and provides a reliable and efficient way to manage configuration compliance objects. This signal deletes any `ConfigCompliance` objects that don't match the current platform. This decision was made to avoid compliance reporting inconsistencies that can arise when outdated or irrelevant objects remain in the database which were generated with the previous platform.
 
 This has a computational impact when updating a Device object's platform. This is similar to the computational impact of an SQL `cascade` option on a delete. This is largely unavoidable and should be limited in impact, such that it will only be the removal of the number of `ConfigCompliance` objects, which is no bigger than the number of  `Config Features`, which is generally intended to be a small amount.
+
+### Configuration Deployment and Remediation
+
+Configuration remediation and deployments of any of the attributes based on the configuration compliance object are calculated based on the last run of the `ConfigCompliance` job. After a configuration deployment to fix any of these attributes (remediation, intended, missing) a new `ConfigCompliance` job must be run before all the compliance results will be updated.
+
+
+### Manual ConfigPlans
+
+When generating a manual `ConfigPlan` the Jinja2 template render has access to Django ORM methods like `.all()`, this also means that methods like `.delete()` can be called, the `render_template` functionality used by Golden Config inherits a Jinja2 Sandbox exception that will block unsafe calls. Golden Config will simply re-raise the exception `jinja2.exceptions.SecurityError: > is not safely callable`.
+
+
+### Hidden Jobs and JobButtons
+
+The configuration deployment and plans features of Golden Config come packaged with Jobs and JobButtons to execute the functionality. In order to to provide a repeatable and consistent behavior these Jobs and JobButtons are designed to only be executed via specialized views. They're not intended to be executed manually from the Jobs/JobButtons menus.
diff --git a/docs/images/config_plan-edit.png b/docs/images/config_plan-edit.png
diff --git a/docs/images/config_plan-generate-filters.png b/docs/images/config_plan-generate-filters.png
diff --git a/docs/images/config_plan-generate-manual.png b/docs/images/config_plan-generate-manual.png
diff --git a/docs/images/config_plan-generate-missing.png b/docs/images/config_plan-generate-missing.png
diff --git a/docs/images/config_plan-view.png b/docs/images/config_plan-view.png
diff --git a/docs/images/remediation_custom_function_setup.png b/docs/images/remediation_custom_function_setup.png
diff --git a/docs/images/remediation_enable_compliance_rule_feature.png b/docs/images/remediation_enable_compliance_rule_feature.png
diff --git a/docs/images/remediation_hier_edit_options.png b/docs/images/remediation_hier_edit_options.png
diff --git a/docs/images/remediation_settings_per_platform.png b/docs/images/remediation_settings_per_platform.png
diff --git a/docs/images/remediation_validate_feature.png b/docs/images/remediation_validate_feature.png
diff --git a/docs/user/app_feature_config_plans.md b/docs/user/app_feature_config_plans.md
@@ -0,0 +1,66 @@
+# Navigating Config Plans
+
+The natural progression for the Golden Config application is providing the ability to execute config deployments. One specific example is to work toward making one or more devices configuration compliant. To aid in this effort, the Golden Config application has the ability to generate plans containing sets of configuration commands from various sources with the intent of deploying them to devices.
+
+The current sources of these plans (i.e. plan types) are as follows:
+
+- The **Intended** configuration(s) of Compliance Feature(s)
+- The **Missing** configuration(s) of Compliance Feature(s)
+- The **Remediation** configuration(s) of Compliance Feature(s) (*)
+- A **Manual** set of configuration commands
+
+!!! note
+    The Intended, Missing and Remediation configuration come from the [Configuration Compliance](./app_feature_compliance.md#compliance-details-view) object that is created when you run the [Perform Configuration Compliance Job](./app_feature_compliance.md#starting-a-compliance-job).
+
+Much like a Configuration Compliance object, each Config Plan is tied directly to a single Device.
+
+## Viewing a Config Plan
+
+You can view a plan by navigating to **Golden Config -> Config Plans** and choosing a generated plan from the list. A Config Plan comprises of the following fields:
+
+- **Device**: The device the plan is to be deployed to.
+- **Date Created**: The date the plan was generated.
+- **Plan Type**: The type of plan used to generate it.
+- **Config Set**: The set of commands to be deployed.
+- **Features** (If Applicable): The Compliance Feature(s) the config set was generated from.
+- **Change Control ID** (Optional): A text field that be used for grouping and filtering plans.
+- **Change Control URL** (Optional): A URL field that can be used to link to an external system tracking change controls.
+- **Job Result**: The Job that generated the plan(s).
+- **Status**: The status of the plan.
+
+![Config Plan View](../images/config_plan-view.png)
+
+## Generating Config Plans
+
+In order to generate a plan, navigate to **Golden Config -> Config Plans** and hit the **Add** button. After choosing the type of plan you want to generate, you can then filter the list of devices you want to generate a Config Plan for by selecting either the list of devices themselves or a by choosing one or more related items such as Location or Status. If you select a plan type that is derived from a Configuration Compliance object, you will have the ability to only generate plans for one or more features, but selecting no features will generate plans for all applicable features.
+
+In addition, you have the ability to specify a Change Control ID & URL that can be associated with all of the plans that will be generated. This can come in handy when it comes to filtering the list of plans to ultimately deploy.
+
+Once you have selected the appropriate options, you can click the **Generate** button which will start a Job to generate the plans.
+
+### Screenshots
+
+![Config Plan Generate Missing](../images/config_plan-generate-missing.png)
+
+![Config Plan Generate Filters](../images/config_plan-generate-filters.png)
+
+![Config Plan Generate Manual](../images/config_plan-generate-manual.png)
+
+### Generating Config Plans via API
+
+The HTTP(S) POST method is not currently enabled for the Config Plan serializer to create plans directly via API. Instead you may run the **GenerateConfigPlans** Job directly via the `plugins/nautobot_golden_config.jobs/GenerateConfigPlans` API endpoint.
+
+## Editing a Config Plan
+
+After a Config Plan is generated you have the ability to edit (or bulk edit) the following fields:
+
+- Change Control ID
+- Change Control URL
+- Status
+- Notes
+- Tags
+
+!!! note
+    You will not be able to modify the Config Set after generation. If it does not contain the desired commands, you will need to delete the plan and recreate it after ensuring the source of the generated commands has been updated.
+
+![Config Plan Edit](../images/config_plan-edit.png)