Merge remote and local app configs after setting git refs (#442)

* Merge remote and local app configs after setting git refs

* Better logging during config merging

* Formatting tweaks

* Add 'ref' property to MockRepoFile

* Move the apps config validation as well

* Remove debug print

* Run config merge tests using with a reference env

bonfire/bonfire.py

@@ -951,6 +951,10 @@ def _get_apps_config(
         log.info("fetching apps config using source: %s", source)
         apps_config = get_appsfile_apps(config)
 
+    # handle git ref/image substitutions if reference environment was provided
+    if ref_env:
+        apps_config = sub_refs(apps_config, ref_env, fallback_ref_env, preferred_params)
+
     # merge remote apps config with local app config
     local_apps = get_local_apps(config)
     apps_config = merge_app_configs(apps_config, local_apps, local_config_method)
@@ -967,10 +971,6 @@ def _get_apps_config(
                 # re-raise with a bit more context
                 raise FatalError(f"{str(err)}, hit on app {app_name}")
 
-    # handle git ref/image substitutions if reference environment was provided
-    if ref_env:
-        apps_config = sub_refs(apps_config, ref_env, fallback_ref_env, preferred_params)
-
     return apps_config
 
 

bonfire/processor.py

@@ -650,6 +650,9 @@ def _get_component_items(self, component_name):
             rf = RepoFile.from_config(component)
             # override template ref if requested
             self._sub_ref(component_name, rf)
+            log.debug(
+                "component: '%s' fetching template using git ref '%s'", component_name, rf.ref
+            )
             commit, template_content = rf.fetch()
         except Exception as err:
             log.error("failed to fetch template file for %s", component_name)

bonfire/qontract.py

@@ -394,7 +394,7 @@ def _find_ref_target_and_update_component(
             final_component["parameters"].update(image_tags)
 
         log.debug(
-            "%s using git ref/image tag from env '%s': %s%s",
+            "%s git ref/image tag found for env '%s': %s%s",
             log_prefix,
             ref_env,
             final_component["ref"],

bonfire/utils.py

@@ -1,9 +1,11 @@
 import atexit
 import copy
+import difflib
 from functools import lru_cache
 import json
 import logging
 import os
+import pprint
 import re
 import shlex
 import socket
@@ -637,14 +639,25 @@ def object_merge(old, new, merge_lists=True):
     return new
 
 
+def _log_diff(old_apps_config, new_apps_config):
+    old_lines = pprint.pformat(old_apps_config).splitlines()
+    new_lines = pprint.pformat(new_apps_config).splitlines()
+    compare_result = difflib.unified_diff(old_lines, new_lines)
+    diff = "\n".join(compare_result)
+    log.info("diff in apps config after merging local config into remote config:\n%s", diff)
+
+
 def merge_app_configs(apps_config, new_apps, method="merge"):
     """
     Merge configurations found in new_apps into apps_config
     """
+    old_apps_config = copy.deepcopy(apps_config)
+
     if method == "override":
         # with this method, any app defined in 'new_apps' completely overrides
         # the config in 'apps_config'
         apps_config.update(new_apps)
+        _log_diff(old_apps_config, apps_config)
         return apps_config
 
     for app_name, new_app_cfg in new_apps.items():
@@ -685,4 +698,5 @@ def merge_app_configs(apps_config, new_apps, method="merge"):
                     f"more than once in app '{app_name}'"
                 )
 
+    _log_diff(old_apps_config, apps_config)
     return apps_config

tests/test_app_configs.py

@@ -384,14 +384,14 @@ def test_local_config_merge(monkeypatch, source):
     actual = _get_apps_config(
         source=source,
         target_env="test_target_env",
-        ref_env=None,
+        ref_env="test_ref_env",
         fallback_ref_env=None,
         local_config_path="na",
         local_config_method="merge",
         preferred_params={},
     )
 
-    expected = _target_apps()
+    expected = _target_apps_w_refs_subbed()
     for _, app_config in expected.items():
         for component in app_config["components"]:
             if component["name"] == "appBcomponent1":
@@ -430,14 +430,14 @@ def test_local_config_override(monkeypatch, source):
     actual = _get_apps_config(
         source=source,
         target_env="test_target_env",
-        ref_env=None,
+        ref_env="test_ref_env",
         fallback_ref_env=None,
         local_config_path="na",
         local_config_method="override",
         preferred_params={},
     )
 
-    expected = _target_apps()
+    expected = _target_apps_w_refs_subbed()
     expected["appB"] = app_b
 
     assert actual == expected
@@ -467,14 +467,14 @@ def test_local_config_merge_update_param(monkeypatch, source):
     actual = _get_apps_config(
         source=source,
         target_env="test_target_env",
-        ref_env=None,
+        ref_env="test_ref_env",
         fallback_ref_env=None,
         local_config_path="na",
         local_config_method="merge",
         preferred_params={},
     )
 
-    expected = _target_apps()
+    expected = _target_apps_w_refs_subbed()
     component1_found = False
     for _, app_config in expected.items():
         for component in app_config["components"]:

tests/test_processor.py

@@ -24,6 +24,10 @@ def from_config(cls, data):
     def add_template(cls, name, template_commit, template_data):
         cls.templates[name] = {"commit": template_commit, "data": template_data}
 
+    @property
+    def ref(self):
+        return self.templates[self.name]["commit"]
+
     def fetch(self):
         return self.templates[self.name]["commit"], self.templates[self.name]["data"]