Merge pull request #79 from Karandash8/54-allow-referencing-variables…

…-in-variables-in-config-file

54 allow referencing variables in variables in config file

README.md

@@ -9,6 +9,7 @@ The idea is that you write your resources in YAML, Jinja2 or Kustomize (includin
 - `App-of-apps` pattern is natively supported
 - Use `include` or `include_raw` in Jinja2 templates to render content of external files
 - Global, per environment and per application Jinja2 variables
+- Variables referencing other variables are supported
 - Source resources can reside in subdirectories
 
 ## Usage
@@ -18,7 +19,9 @@ python3 -m venv .venv
 pip install make-argocd-fly
 
 make-argocd-fly -h
-usage: main.py [-h] [--root-dir ROOT_DIR] [--config-file CONFIG_FILE] [--source-dir SOURCE_DIR] [--output-dir OUTPUT_DIR] [--tmp-dir TMP_DIR] [--render-apps RENDER_APPS] [--render-envs RENDER_ENVS] [--skip-generate] [--preserve-tmp-dir] [--clean] [--print-vars] [--yaml-linter] [--kube-linter] [--loglevel LOGLEVEL]
+usage: main.py [-h] [--root-dir ROOT_DIR] [--config-file CONFIG_FILE] [--source-dir SOURCE_DIR] [--output-dir OUTPUT_DIR] [--tmp-dir TMP_DIR] [--render-apps RENDER_APPS]
+               [--render-envs RENDER_ENVS] [--skip-generate] [--preserve-tmp-dir] [--clean] [--print-vars] [--var-identifier VAR_IDENTIFIER] [--yaml-linter] [--kube-linter]
+               [--loglevel LOGLEVEL]
 
 Render ArgoCD Applications.
 
@@ -40,6 +43,8 @@ options:
   --preserve-tmp-dir    Preserve temporary directory
   --clean               Clean all applications in output directory
   --print-vars          Print variables for each application
+  --var-identifier VAR_IDENTIFIER
+                        Variable prefix in config.yml file (default: $)
   --yaml-linter         Run yamllint against output directory (https://github.com/adrienverge/yamllint)
   --kube-linter         Run kube-linter against output directory (https://github.com/stackrox/kube-linter)
   --loglevel LOGLEVEL   DEBUG, INFO, WARNING, ERROR, CRITICAL
@@ -50,14 +55,20 @@ options:
 Example configuration file:
 ```tests/manual/config.yml```
 
+### Source directory structure
+Example directory structure:
+```tests/manual/source```
+
+When kustomization overlays are used, kustomization base directory shall be named `base`, overlay directories shall be named after the corresponding environments names.
+
 ### app parameters
 - `app_deployer` - name of the application that will deploy this application
 - `project` - ArgoCD project name
 - `destination_namespace` - namespace where the application will be deployed
 - `app_deployer_env` - (OPTIONAL) environment of the application that will deploy this application
 - `vars` - (OPTIONAL) application specific jinja2 variables
 
-## Jinja2 extensions
+### Jinja2 extensions
 To render a template in the current jinja2 template, use the following block:
 
 ```
@@ -82,20 +93,6 @@ To perform a DNS lookup, use the following filter:
 
 Ansible filters are supported as well: https://pypi.org/project/jinja2-ansible-filters/
 
-## Caveats
-### Expectations
-- `kustomize` and `helm` are expected to be installed locally.
-- `kube-linter` is expected to be installed locally (https://github.com/stackrox/kube-linter).
-- `libyaml` is expected to be installed locally for speeding up YAMLs generation.
-- Comments are not rendered in the final output manifests.
-
-### Source directory structure
-
-Example directory structure:
-```tests/manual/source```
-
-When kustomization overlays are used, kustomization base directory shall be named `base`, overlay directories shall be named after the corresponding environments names.
-
 ### kustomization.yml
 Files referenced in the `resources` section shall be named after Kubernetes resource type + `_` + resource name. Example:
 
@@ -113,6 +110,14 @@ The folloving variable names are reserved (at the root level) and shall not be u
 - env_name
 - app_name
 
+### Referencing variables in config.yml
+Variables can be referenced in the configuration file (including *app parameters*) using the following syntax:
+```${var_name}``` and  ```${var_name[subvar_name][...]}```.
+
+Variables can also be used as substring values:
+```prefix-${var_name}-suffix```.
+
+
 ### Expected variables
 The folloving variables are expected to be provided:
 - argocd.api_server
@@ -125,6 +130,12 @@ The folloving variables are expected to be provided:
 - argocd.finalizers
 - argocd.ignoreDifferences
 
+## Caveats
+- `kustomize` and `helm` are expected to be installed locally.
+- `kube-linter` is expected to be installed locally (https://github.com/stackrox/kube-linter).
+- `libyaml` is expected to be installed locally for speeding up YAMLs generation.
+- Comments are not rendered in the final output manifests.
+
 ## For developers
 ### Build instructions
 https://setuptools.pypa.io/en/latest/userguide/quickstart.html

make_argocd_fly/application.py

@@ -7,7 +7,8 @@
 
 from make_argocd_fly.resource import ResourceViewer, ResourceWriter
 from make_argocd_fly.renderer import JinjaRenderer
-from make_argocd_fly.utils import multi_resource_parser, resource_parser, merge_dicts, generate_filename
+from make_argocd_fly.utils import multi_resource_parser, resource_parser, merge_dicts, generate_filename, \
+  VarsResolver
 from make_argocd_fly.config import get_config
 from make_argocd_fly.cli_args import get_cli_args
 
@@ -88,16 +89,20 @@ async def generate_resources(self) -> None:
     renderer = JinjaRenderer()
 
     for (app_name, env_name, project, destination_namespace) in self._find_deploying_apps(self.app_name, self.env_name):
-      template_vars = merge_dicts(self.config.get_vars(), self.config.get_env_vars(env_name), self.config.get_app_vars(env_name, app_name), {
-        '__application': {
-          'application_name': '-'.join([os.path.basename(app_name), env_name]).replace('_', '-'),
-          'path': os.path.join(os.path.basename(self._config.get_output_dir()), env_name, app_name),
-          'project': project,
-          'destination_namespace': destination_namespace
-        },
-        'env_name': env_name,
-        'app_name': app_name
-      })
+      template_vars = VarsResolver.resolve_all(merge_dicts(self.config.get_vars(), self.config.get_env_vars(env_name),
+                                                           self.config.get_app_vars(env_name, app_name), {
+                                                           '__application': {
+                                                             'application_name': '-'.join([os.path.basename(app_name), env_name]).replace('_', '-'),
+                                                             'path': os.path.join(os.path.basename(self._config.get_output_dir()),
+                                                                                  env_name, app_name
+                                                                                  ),
+                                                             'project': project,
+                                                             'destination_namespace': destination_namespace
+                                                           },
+                                                           'env_name': env_name,
+                                                           'app_name': app_name}),
+                                               var_identifier=self.cli_args.get_var_identifier())
+
       content = renderer.render(textwrap.dedent(self.APPLICATION_RESOUCE_TEMPLATE), template_vars)
       resources.append(content)
 
@@ -116,9 +121,10 @@ async def generate_resources(self) -> None:
 
     resources = []
     renderer = JinjaRenderer(self.app_viewer)
-    template_vars = merge_dicts(self.config.get_vars(), self.config.get_env_vars(self.env_name),
-                                self.config.get_app_vars(self.env_name, self.app_name), {'env_name': self.env_name, 'app_name': self.app_name}
-                                )
+    template_vars = VarsResolver.resolve_all(merge_dicts(self.config.get_vars(), self.config.get_env_vars(self.env_name),
+                                                         self.config.get_app_vars(self.env_name, self.app_name),
+                                                         {'env_name': self.env_name, 'app_name': self.app_name}),
+                                             var_identifier=self.cli_args.get_var_identifier())
     if self.cli_args.get_print_vars():
       log.info('Variables for application {} in environment {}:\n{}'.format(self.app_name, self.env_name, pformat(template_vars)))
 
@@ -148,7 +154,6 @@ async def _run_kustomize(self, dir_path: str, retries: int = 3) -> str:
         stderr=asyncio.subprocess.PIPE)
 
       stdout, stderr = await proc.communicate()
-
       if stderr:
         log.error('Kustomize error: {}'.format(stderr))
         log.info('Retrying {}/{}'.format(attempt + 1, retries))
@@ -165,9 +170,11 @@ async def prepare(self) -> str:
 
     tmp_resource_writer = ResourceWriter(tmp_dir)
     renderer = JinjaRenderer(self.app_viewer)
-    template_vars = merge_dicts(self.config.get_vars(), self.config.get_env_vars(self.env_name),
-                                self.config.get_app_vars(self.env_name, self.app_name), {'env_name': self.env_name, 'app_name': self.app_name}
-                                )
+    template_vars = VarsResolver.resolve_all(merge_dicts(self.config.get_vars(), self.config.get_env_vars(self.env_name),
+                                                         self.config.get_app_vars(self.env_name, self.app_name),
+                                                         {'env_name': self.env_name, 'app_name': self.app_name}),
+                                             var_identifier=self.cli_args.get_var_identifier()
+                                             )
     if self.cli_args.get_print_vars():
       log.info('Variables for application {} in environment {}:\n{}'.format(self.app_name, self.env_name, pformat(template_vars)))
 

make_argocd_fly/cli_args.py

@@ -17,6 +17,7 @@ def __init__(self) -> None:
     self.preserve_tmp_dir = None
     self.clean = None
     self.print_vars = None
+    self.var_identifier = None
     self.yaml_linter = None
     self.kube_linter = None
     self.loglevel = None
@@ -75,6 +76,11 @@ def get_print_vars(self):
       raise Exception("print_vars is not set")
     return self.print_vars
 
+  def get_var_identifier(self):
+    if self.var_identifier is None:
+      raise Exception("var_identifier is not set")
+    return self.var_identifier
+
   def get_yaml_linter(self):
     if self.yaml_linter is None:
       raise Exception("yaml_linter is not set")

make_argocd_fly/main.py

@@ -71,6 +71,8 @@ async def generate() -> None:
     log.info('Rendering resources')
     await asyncio.gather(*[asyncio.create_task(app.generate_resources()) for app in apps])
   except Exception:
+    for task in asyncio.all_tasks():
+      task.cancel()
     raise
 
   output_writer = ResourceWriter(config.get_output_dir())
@@ -106,6 +108,7 @@ def main() -> None:
   parser.add_argument('--preserve-tmp-dir', action='store_true', help='Preserve temporary directory')
   parser.add_argument('--clean', action='store_true', help='Clean all applications in output directory')
   parser.add_argument('--print-vars', action='store_true', help='Print variables for each application')
+  parser.add_argument('--var-identifier', type=str, default='$', help='Variable prefix in config.yml file (default: $)')
   parser.add_argument('--yaml-linter', action='store_true', help='Run yamllint against output directory (https://github.com/adrienverge/yamllint)')
   parser.add_argument('--kube-linter', action='store_true', help='Run kube-linter against output directory (https://github.com/stackrox/kube-linter)')
   parser.add_argument('--loglevel', type=str, default='INFO', help='DEBUG, INFO, WARNING, ERROR, CRITICAL')

make_argocd_fly/utils.py

@@ -1,9 +1,89 @@
 import logging
 import re
+import copy
+import ast
 
 log = logging.getLogger(__name__)
 
 
+class VarsResolver:
+  def __init__(self, var_identifier: str = '$') -> None:
+    self.var_identifier = var_identifier
+    self.resolution_counter = 0
+
+  def _find_var_position(self, value: str, start: int = 0) -> tuple[int, int]:
+    var_start = value.find(self.var_identifier, start)
+    if var_start == -1 or value[var_start + 1] != '{':
+      return (-1, -1)
+
+    var_end = value.find('}', var_start)
+    if var_end == -1:
+      return (-1, -1)
+
+    return (var_start + 1, var_end)
+
+  def _resolve_value(self, vars: dict, value: str) -> str:
+    resolved_value = ''
+    try:
+      start = 0
+      (var_start, var_end) = self._find_var_position(value, start)
+
+      if (var_start, var_end) == (-1, -1):
+        return value
+
+      while (var_start, var_end) != (-1, -1):
+        if (var_start - 1) > start:
+          resolved_value += value[start:var_start - 1]
+
+        resolved_value += value[var_start:var_end + 1].format(**vars)
+        self.resolution_counter += 1
+        start = var_end + 1
+
+        (var_start, var_end) = self._find_var_position(value, start)
+
+      resolved_value += value[start:]
+
+      try:
+        resolved_value = ast.literal_eval(resolved_value)
+      except (SyntaxError, ValueError):
+        pass
+
+      return resolved_value
+    except KeyError:
+      log.error('Variable {} not found in vars'.format(value[var_start - 1:var_end + 1]))
+      raise
+
+  def _iterate(self, vars: dict, value=None, initial=True):
+    value = value or vars if initial else value
+    if isinstance(value, dict):
+      for k, v in value.items():
+        value[k] = self._iterate(vars, v, False)
+    elif isinstance(value, list):
+      for idx, i in enumerate(value):
+        value[idx] = self._iterate(vars, i, False)
+    elif isinstance(value, str):
+      value = self._resolve_value(vars, value)
+    return value
+
+  def get_resolutions(self) -> int:
+    return self.resolution_counter
+
+  def resolve(self, vars: dict) -> dict:
+    self.resolution_counter = 0
+
+    return self._iterate(copy.deepcopy(vars))
+
+  @staticmethod
+  def resolve_all(vars: dict, var_identifier: str = '$') -> dict:
+      resolver = VarsResolver(var_identifier)
+
+      resolved_vars = resolver.resolve(vars)
+      while resolver.get_resolutions() > 0:
+        resolved_vars = resolver.resolve(resolved_vars)
+
+      return resolved_vars
+
+
 # TODO: rename this, logic does not align with multi_resource_parser
 def resource_parser(resource_yml: str) -> tuple[str, str]:
   resource_kind = None

tests/manual/config.yml

@@ -29,8 +29,8 @@ envs:
                 jqPathExpressions:
                   - .spec.seLinuxMount
             finalizers: null
-      app_3: {app_deployer: service_deployer, project: management, destination_namespace: kube-default}
-      app_4: {app_deployer: service_deployer, project: management, destination_namespace: kube-default}
+      app_3: {app_deployer: service_deployer, project: management, destination_namespace: "${namespace}"}
+      app_4: {app_deployer: service_deployer, project: management, destination_namespace: kube-default, vars: {argocd: {ignoreDifferences: "${argocd_ignore_diff}"}}}
       app_5: {app_deployer: service_deployer, project: management, destination_namespace: kube-default}
       app_6: {app_deployer: service_deployer, project: management, destination_namespace: kube-default}
       subdirectory/app_7: {app_deployer: service_deployer, project: management, destination_namespace: kube-default}
@@ -77,9 +77,19 @@ vars:
         - ServerSideApply=true
     finalizers:
       - resources-finalizer.argocd.argoproj.io
-  namespace: kube-system
+  namespace: kube-default
   version: 0.1.0
-  # reference_version: '["vars"]["version"]'
+  double_reference_version: ${reference_version}
+  reference_version: ${app[version]}
+  resource: Deployment_thanos.yml
   app:
-    resource: Deployment_thanos.yml
+    resource: ${resource}
+    version: 0.1.0
   json_var: json
+  argocd_ignore_diff:
+    - group: apps
+      kind: Deployment
+      name: guestbook
+      namespace: default
+      jsonPointers:
+        - /spec/replicas

tests/manual/output/management/app_4/Deployment_thanos-sidecar-2.yml

@@ -3,7 +3,7 @@ kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: thanos-sidecar-2
-  namespace: kube-system
+  namespace: kube-default
 data: |-
   {
     "test": "jsoninclude",

tests/manual/output/management/app_4/Deployment_thanos-sidecar.yml

@@ -3,4 +3,4 @@ kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: thanos-sidecar
-  namespace: kube-system
+  namespace: kube-default

tests/manual/output/management/service_deployer/Application_app-4-management.yml

@@ -18,3 +18,10 @@ spec:
   syncPolicy:
     syncOptions:
       - ServerSideApply=true
+  ignoreDifferences:
+    - group: apps
+      jsonPointers:
+        - /spec/replicas
+      kind: Deployment
+      name: guestbook
+      namespace: default

tests/test_resource.py

@@ -70,7 +70,7 @@ def test_ResourceViewer_build_with_non_existing_path(tmp_path, caplog):
   resource_viewer = ResourceViewer(str(dir_root), element_path)
 
   with pytest.raises(Exception):
-      resource_viewer.build()
+    resource_viewer.build()
   assert 'Path does not exist' in caplog.text
 
 def test_ResourceViewer_build_with_directories_and_files(tmp_path):