diff --git a/README.md b/README.md index 1684b3e..8665dad 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ envs: : app_deployer: ## application that will deploy this application app_deployer_env: ## (OPTIONAL) for multi-environments with single ArgoCD deployment + non_k8s_files_to_render: [] ## (OPTIONAL) list of files to render that are not Kubernetes resources (e.g., values.yml) project: ## ArgoCD project name destination_namespace: ## default namespace where the application resources will be deployed vars: @@ -136,7 +137,7 @@ vars: ``` ### Variables in `config.yml` -Variables can be referenced in the configuration file (including in the application parameters section) using the following syntax: +Variables can be referenced in the configuration file using the following syntax: ```${var_name}``` and ```${var_name[dict_key][...]}```. Variables can also be used as substring values: @@ -147,7 +148,7 @@ To include file content in the current Jinja2 template, use the following block: ``` {%- filter indent(width=4) %} -{% include_raw 'app_4/files/file.json' %} +{% include_raw 'files/file.json' %} {% endfilter %} ``` @@ -158,7 +159,7 @@ To render a template in the current jinja2 template, use the following block: ``` {%- filter indent(width=4) %} -{% include 'app_5/files/file.json.j2' %} +{% include 'files/file.json.j2' %} {% endfilter %} ``` diff --git a/dev_requirements.txt b/dev_requirements.txt index 6647917..f154119 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -2,5 +2,6 @@ setuptools build==0.10.0 twine==4.0.2 pytest==7.4.0 +pytest-asyncio==0.23.8 yelp-gprof2dot==1.2.0 flake8==7.0.0 diff --git a/make_argocd_fly/application.py b/make_argocd_fly/application.py index 49e95a5..ac1fa94 100644 --- a/make_argocd_fly/application.py +++ b/make_argocd_fly/application.py @@ -1,21 +1,21 @@ import logging import os -import asyncio from abc import ABC, abstractmethod import textwrap from pprint import pformat -from make_argocd_fly.resource import ResourceViewer, ResourceWriter -from make_argocd_fly.renderer import JinjaRenderer -from make_argocd_fly.utils import multi_resource_parser, resource_parser, merge_dicts, generate_filename, \ - VarsResolver +from make_argocd_fly.resource import ResourceViewer +from make_argocd_fly.utils import merge_dicts, VarsResolver, get_app_rel_path from make_argocd_fly.config import get_config from make_argocd_fly.cli_args import get_cli_args +from make_argocd_fly.steps import FindAppsStep, RenderYamlStep, RenderJinjaFromViewerStep, RenderJinjaFromMemoryStep, \ + WriteResourcesStep, ReadSourceStep, RunKustomizeStep +from make_argocd_fly.exceptions import MissingSourceResourcesError log = logging.getLogger(__name__) -class AbstractApplication(ABC): +class AbstractWorkflow(ABC): def __init__(self, app_name: str, env_name: str, app_viewer: ResourceViewer = None) -> None: super().__init__() @@ -24,20 +24,13 @@ def __init__(self, app_name: str, env_name: str, app_viewer: ResourceViewer = No self.app_viewer = app_viewer self.config = get_config() self.cli_args = get_cli_args() - self.resources = None - - async def prepare(self) -> str: - pass @abstractmethod - async def generate_resources(self) -> None: + async def process(self) -> None: pass - def get_app_rel_path(self) -> str: - return os.path.join(self.env_name, self.app_name) - -class AppOfApps(AbstractApplication): +class AppOfAppsWorkflow(AbstractWorkflow): APPLICATION_RESOUCE_TEMPLATE = '''\ apiVersion: argoproj.io/v1alpha1 kind: Application @@ -68,32 +61,25 @@ class AppOfApps(AbstractApplication): ''' def __init__(self, app_name: str, env_name: str, app_viewer: ResourceViewer = None) -> None: - self._config = get_config() super().__init__(app_name, env_name, app_viewer) + self.find_apps_step = FindAppsStep() + self.render_jinja_step = RenderJinjaFromMemoryStep() + self.write_resources_step = WriteResourcesStep() - log.debug('Created application {} of type {} for environment {}'.format(app_name, __class__.__name__, env_name)) - - def _find_deploying_apps(self, app_deployer_name: str, app_deployer_env_name: str) -> tuple[str, str, str, str]: - for env_name, env_data in self._config.get_envs().items(): - for app_name, app_data in env_data['apps'].items(): - if 'app_deployer' in app_data and 'project' in app_data and 'destination_namespace' in app_data: - if (app_deployer_name == app_data['app_deployer'] and - (('app_deployer_env' not in app_data and env_name == app_deployer_env_name) or - ('app_deployer_env' in app_data and app_deployer_env_name == app_data['app_deployer_env']))): - yield (app_name, env_name, app_data['project'], app_data['destination_namespace']) + log.debug('Created application {} with {} for environment {}'.format(app_name, __class__.__name__, env_name)) - async def generate_resources(self) -> None: - log.debug('Generating resources for application {} in environment {}'.format(self.app_name, self.env_name)) + async def process(self) -> None: + log.debug('Starting to process application {} in environment {}'.format(self.app_name, self.env_name)) - resources = [] - renderer = JinjaRenderer() + self.find_apps_step.configure(self.app_name, self.env_name) + await self.find_apps_step.run() - for (app_name, env_name, project, destination_namespace) in self._find_deploying_apps(self.app_name, self.env_name): + for (app_name, env_name, project, destination_namespace) in self.find_apps_step.get_apps(): template_vars = VarsResolver.resolve_all(merge_dicts(self.config.get_vars(), self.config.get_env_vars(env_name), self.config.get_app_vars(env_name, app_name), { '__application': { 'application_name': '-'.join([os.path.basename(app_name), env_name]).replace('_', '-'), - 'path': os.path.join(os.path.basename(self._config.get_output_dir()), + 'path': os.path.join(os.path.basename(self.config.get_output_dir()), env_name, app_name ), 'project': project, @@ -103,24 +89,28 @@ async def generate_resources(self) -> None: 'app_name': app_name}), var_identifier=self.cli_args.get_var_identifier()) - content = renderer.render(textwrap.dedent(self.APPLICATION_RESOUCE_TEMPLATE), template_vars) - resources.append(content) + self.render_jinja_step.configure(textwrap.dedent(self.APPLICATION_RESOUCE_TEMPLATE), self.app_name, self.env_name, template_vars) + await self.render_jinja_step.run() - self.resources = '\n---\n'.join(resources) log.debug('Generated resources for application {} in environment {}'.format(self.app_name, self.env_name)) + self.write_resources_step.configure(self.config.get_output_dir(), self.render_jinja_step.get_resources()) + await self.write_resources_step.run() + log.info('Updated application {} in environment {}'.format(self.app_name, self.env_name)) -class Application(AbstractApplication): + +class SimpleWorkflow(AbstractWorkflow): def __init__(self, app_name: str, env_name: str, app_viewer: ResourceViewer = None) -> None: super().__init__(app_name, env_name, app_viewer) + self.render_yaml_step = RenderYamlStep() + self.render_jinja_step = RenderJinjaFromViewerStep(app_viewer) + self.write_resources_step = WriteResourcesStep() - log.debug('Created application {} of type {} for environment {}'.format(app_name, __class__.__name__, env_name)) + log.debug('Created application {} with {} for environment {}'.format(app_name, __class__.__name__, env_name)) - async def generate_resources(self) -> None: - log.debug('Generating resources for application {} in environment {}'.format(self.app_name, self.env_name)) + async def process(self) -> None: + log.debug('Starting to process application {} in environment {}'.format(self.app_name, self.env_name)) - resources = [] - renderer = JinjaRenderer(self.app_viewer) template_vars = VarsResolver.resolve_all(merge_dicts(self.config.get_vars(), self.config.get_env_vars(self.env_name), self.config.get_app_vars(self.env_name, self.app_name), {'env_name': self.env_name, 'app_name': self.app_name}), @@ -128,48 +118,37 @@ async def generate_resources(self) -> None: if self.cli_args.get_print_vars(): log.info('Variables for application {} in environment {}:\n{}'.format(self.app_name, self.env_name, pformat(template_vars))) - yml_children = self.app_viewer.get_files_children(r'(\.yml|\.yml\.j2)$') - for yml_child in yml_children: - content = yml_child.content - if yml_child.element_rel_path.endswith('.j2'): - content = renderer.render(content, template_vars, yml_child.element_rel_path) + yml_children = self.app_viewer.get_files_children(r'(\.yml)$') + self.render_yaml_step.configure(yml_children, self.app_name, self.env_name) + await self.render_yaml_step.run() - resources.append(content) + j2_children = self.app_viewer.get_files_children(r'(\.yml\.j2)$') + self.render_jinja_step.configure(j2_children, self.app_name, self.env_name, template_vars) + await self.render_jinja_step.run() - self.resources = '\n---\n'.join(resources) log.debug('Generated resources for application {} in environment {}'.format(self.app_name, self.env_name)) + self.write_resources_step.configure(self.config.get_output_dir(), self.render_yaml_step.get_resources() + self.render_jinja_step.get_resources()) + await self.write_resources_step.run() + + log.info('Updated application {} in environment {}'.format(self.app_name, self.env_name)) + -class KustomizeApplication(AbstractApplication): +class KustomizeWorkflow(AbstractWorkflow): def __init__(self, app_name: str, env_name: str, app_viewer: ResourceViewer = None) -> None: super().__init__(app_name, env_name, app_viewer) + self.render_yaml_step = RenderYamlStep() + self.render_jinja_step = RenderJinjaFromViewerStep(app_viewer) + self.tmp_write_resources_step = WriteResourcesStep() + self.tmp_read_source_step = ReadSourceStep() + self.run_kustomize_step = RunKustomizeStep() + self.write_resources_step = WriteResourcesStep() - log.debug('Created application {} of type {} for environment {}'.format(app_name, __class__.__name__, env_name)) - - async def _run_kustomize(self, dir_path: str, retries: int = 3) -> str: - for attempt in range(retries): - proc = await asyncio.create_subprocess_shell( - 'kustomize build --enable-helm {}'.format(dir_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) - - stdout, stderr = await proc.communicate() - if stderr: - log.error('Kustomize error: {}'.format(stderr)) - log.info('Retrying {}/{}'.format(attempt + 1, retries)) - continue - break - else: - raise Exception('Kustomize execution failed for application {} in environment {}'.format(self.app_name, self.env_name)) - - return stdout.decode("utf-8") + log.debug('Created application {} with {} for environment {}'.format(app_name, __class__.__name__, env_name)) - async def prepare(self) -> str: - config = get_config() - tmp_dir = config.get_tmp_dir() + async def process(self) -> None: + log.debug('Starting to process application {} in environment {}'.format(self.app_name, self.env_name)) - tmp_resource_writer = ResourceWriter(tmp_dir) - renderer = JinjaRenderer(self.app_viewer) template_vars = VarsResolver.resolve_all(merge_dicts(self.config.get_vars(), self.config.get_env_vars(self.env_name), self.config.get_app_vars(self.env_name, self.app_name), {'env_name': self.env_name, 'app_name': self.app_name}), @@ -178,67 +157,54 @@ async def prepare(self) -> str: if self.cli_args.get_print_vars(): log.info('Variables for application {} in environment {}:\n{}'.format(self.app_name, self.env_name, pformat(template_vars))) - yml_children = self.app_viewer.get_files_children(r'(\.yml|\.yml\.j2)$', ['base', self.env_name]) - for yml_child in yml_children: - content = yml_child.content - if yml_child.element_rel_path.endswith('.j2'): - try: - content = renderer.render(content, template_vars, yml_child.element_rel_path) - except Exception as e: - log.error('Error rendering template {}: {}'.format(yml_child.element_rel_path, e)) - raise - - # TODO: (None, None) is not a good way to check if the content is a k8s resource - if resource_parser(content) != (None, None): - for resource_kind, resource_name, resource_yml in multi_resource_parser(content): - file_path = os.path.join(self.env_name, os.path.dirname(yml_child.element_rel_path), generate_filename([resource_kind, resource_name])) - tmp_resource_writer.store_resource(file_path, resource_yml) - else: - file_path = os.path.join( - self.env_name, os.path.dirname(yml_child.element_rel_path), - generate_filename([os.path.basename(yml_child.element_rel_path.split('.')[0])]) - ) - tmp_resource_writer.store_resource(file_path, content) - - await tmp_resource_writer.write_resources() - - async def generate_resources(self) -> None: - log.debug('Generating resources for application {} in environment {}'.format(self.app_name, self.env_name)) - - config = get_config() - tmp_dir = config.get_tmp_dir() - - tmp_source_viewer = ResourceViewer(os.path.join(tmp_dir, self.get_app_rel_path())) - tmp_source_viewer.build() - - yml_child = tmp_source_viewer.get_element(os.path.join(self.env_name, 'kustomization.yml')) - if yml_child: - self.resources = await self._run_kustomize(os.path.join(tmp_source_viewer.root_element_abs_path, os.path.dirname(yml_child.element_rel_path))) - log.debug('Generated resources for application {} in environment {}'.format(self.app_name, self.env_name)) - return - - yml_child = tmp_source_viewer.get_element(os.path.join('base', 'kustomization.yml')) - if yml_child: - self.resources = await self._run_kustomize(os.path.join(tmp_source_viewer.root_element_abs_path, os.path.dirname(yml_child.element_rel_path))) - log.debug('Generated resources for application {} in environment {}'.format(self.app_name, self.env_name)) - return - - yml_child = tmp_source_viewer.get_element('kustomization.yml') - if yml_child: - self.resources = await self._run_kustomize(os.path.join(tmp_source_viewer.root_element_abs_path, os.path.dirname(yml_child.element_rel_path))) - log.debug('Generated resources for application {} in environment {}'.format(self.app_name, self.env_name)) - return - - log.error('Missing kustomization.yml in the application directory. Skipping application') - - -def application_factory(app_viewer: ResourceViewer, app_name: str, env_name: str) -> AbstractApplication: - if app_viewer: - kustomize_children = app_viewer.get_files_children('kustomization.yml') + yml_children = self.app_viewer.get_files_children(r'(\.yml)$', ['base', self.env_name]) + self.render_yaml_step.configure(yml_children, self.app_name, self.env_name) + await self.render_yaml_step.run() + + j2_children = self.app_viewer.get_files_children(r'(\.yml\.j2)$', ['base', self.env_name]) + self.render_jinja_step.configure(j2_children, self.app_name, self.env_name, template_vars) + await self.render_jinja_step.run() + + self.tmp_write_resources_step.configure(self.config.get_tmp_dir(), self.render_yaml_step.get_resources() + self.render_jinja_step.get_resources()) + await self.tmp_write_resources_step.run() + + self.tmp_read_source_step.configure(os.path.join(self.config.get_tmp_dir(), get_app_rel_path(self.app_name, self.env_name))) + await self.tmp_read_source_step.run() + + self.run_kustomize_step.configure(self.tmp_read_source_step.get_viewer(), self.app_name, self.env_name) + await self.run_kustomize_step.run() + + log.debug('Generated resources for application {} in environment {}'.format(self.app_name, self.env_name)) + + self.write_resources_step.configure(self.config.get_output_dir(), self.run_kustomize_step.get_resources()) + await self.write_resources_step.run() + + log.info('Updated application {} in environment {}'.format(self.app_name, self.env_name)) + + +async def workflow_factory(app_name: str, env_name: str, source_path: str) -> AbstractWorkflow: + read_source_step = ReadSourceStep() + read_source_step.configure(source_path) + + try: + await read_source_step.run() + viewer = read_source_step.get_viewer() + + kustomize_children = viewer.get_files_children('kustomization.yml') if not kustomize_children: - return Application(app_name, env_name, app_viewer) + return SimpleWorkflow(app_name, env_name, viewer) else: - return KustomizeApplication(app_name, env_name, app_viewer) - else: - return AppOfApps(app_name, env_name, None) + return KustomizeWorkflow(app_name, env_name, viewer) + except MissingSourceResourcesError: + return AppOfAppsWorkflow(app_name, env_name) + + +class Application(): + def __init__(self, app_name: str, env_name: str, workflow: AbstractWorkflow) -> None: + self.app_name = app_name + self.env_name = env_name + self.workflow = workflow + + async def process(self) -> None: + await self.workflow.process() diff --git a/make_argocd_fly/config.py b/make_argocd_fly/config.py index 2527a0d..1b4149b 100644 --- a/make_argocd_fly/config.py +++ b/make_argocd_fly/config.py @@ -41,13 +41,13 @@ def get_tmp_dir(self) -> str: return get_abs_path(self.root_dir, self.tmp_dir, allow_missing=True) def get_envs(self) -> dict: - if not self.envs: + if not isinstance(self.envs, dict): log.error('Config was not initialized.') raise Exception return self.envs def get_vars(self) -> dict: - if not self.envs: + if not isinstance(self.vars, dict): log.error('Config was not initialized.') raise Exception return self.vars @@ -71,6 +71,23 @@ def get_app_vars(self, env_name: str, app_name: str) -> dict: return envs[env_name]['apps'][app_name]['vars'] if 'vars' in envs[env_name]['apps'][app_name] else {} + def get_app_params(self, env_name: str, app_name: str) -> dict: + envs = self.get_envs() + if env_name not in envs: + log.error('Environment {} is not defined'.format(env_name)) + raise Exception + + if app_name not in envs[env_name]['apps']: + log.error('Application {} is not defined in environment {}'.format(app_name, env_name)) + raise Exception + + params = {} + for key, value in envs[env_name]['apps'][app_name].items(): + if key != 'vars': + params[key] = value + + return params + config = Config() diff --git a/make_argocd_fly/exceptions.py b/make_argocd_fly/exceptions.py new file mode 100644 index 0000000..440da40 --- /dev/null +++ b/make_argocd_fly/exceptions.py @@ -0,0 +1,2 @@ +class MissingSourceResourcesError(Exception): + pass diff --git a/make_argocd_fly/main.py b/make_argocd_fly/main.py index 82b4e5b..5aa5c65 100644 --- a/make_argocd_fly/main.py +++ b/make_argocd_fly/main.py @@ -11,9 +11,8 @@ from make_argocd_fly.cli_args import populate_cli_args, get_cli_args from make_argocd_fly.config import read_config, get_config, LOG_CONFIG_FILE, CONFIG_FILE, \ SOURCE_DIR, OUTPUT_DIR, TMP_DIR -from make_argocd_fly.utils import multi_resource_parser, generate_filename, latest_version_check -from make_argocd_fly.resource import ResourceViewer, ResourceWriter -from make_argocd_fly.application import application_factory +from make_argocd_fly.utils import latest_version_check +from make_argocd_fly.application import workflow_factory, Application logging.basicConfig(level='INFO') @@ -32,12 +31,11 @@ def init_logging(loglevel: str) -> None: pass -def create_applications(render_apps, render_envs): +async def generate() -> None: config = get_config() - - log.info('Reading source directory') - source_viewer = ResourceViewer(config.get_source_dir()) - source_viewer.build() + cli_args = get_cli_args() + render_apps = cli_args.get_render_apps() + render_envs = cli_args.get_render_envs() apps_to_render = render_apps.split(',') if render_apps is not None else [] envs_to_render = render_envs.split(',') if render_envs is not None else [] @@ -52,47 +50,17 @@ def create_applications(render_apps, render_envs): if apps_to_render and app_name not in apps_to_render: continue - app_viewer = source_viewer.get_element(app_name) - apps.append(application_factory(app_viewer, app_name, env_name)) - - return apps - - -async def generate() -> None: - cli_args = get_cli_args() - config = get_config() - - apps = create_applications(cli_args.get_render_apps(), cli_args.get_render_envs()) + workflow = await workflow_factory(app_name, env_name, os.path.join(config.get_source_dir(), app_name)) + apps.append(Application(app_name, env_name, workflow)) + log.info('Processing applications') try: - log.info('Generating temporary files') - await asyncio.gather(*[asyncio.create_task(app.prepare()) for app in apps]) - - log.info('Rendering resources') - await asyncio.gather(*[asyncio.create_task(app.generate_resources()) for app in apps]) + await asyncio.gather(*[asyncio.create_task(app.process()) for app in apps]) except Exception: for task in asyncio.all_tasks(): task.cancel() raise - output_writer = ResourceWriter(config.get_output_dir()) - for app in apps: - for resource_kind, resource_name, resource_yml in multi_resource_parser(app.resources): - file_path = os.path.join(app.get_app_rel_path(), generate_filename([resource_kind, resource_name])) - output_writer.store_resource(file_path, resource_yml) - - if apps: - log.info('The following applications have been updated:') - for app in apps: - app_dir = os.path.join(config.get_output_dir(), app.get_app_rel_path()) - log.info('Environment: {}, Application: {}, Path: {}'.format(app.env_name, app.app_name, app_dir)) - if os.path.exists(app_dir): - shutil.rmtree(app_dir) - - log.info('Writing resources files') - os.makedirs(config.get_output_dir(), exist_ok=True) - await output_writer.write_resources() - def main() -> None: parser = argparse.ArgumentParser(description='Render ArgoCD Applications.') diff --git a/make_argocd_fly/renderer.py b/make_argocd_fly/renderer.py index c1b4c8a..00788a4 100644 --- a/make_argocd_fly/renderer.py +++ b/make_argocd_fly/renderer.py @@ -13,7 +13,7 @@ class AbstractRenderer(ABC): @abstractmethod - def render(self, content: str, template_vars: dict = None) -> str: + def render(self, content: str) -> str: pass @@ -21,7 +21,7 @@ class DummyRenderer(AbstractRenderer): def __init__(self) -> None: pass - def render(self, content: str, template_vars: dict = None) -> str: + def render(self, content: str) -> str: return content @@ -63,6 +63,9 @@ def __init__(self, viewer: ResourceViewer = None) -> None: 'jinja2_ansible_filters.AnsibleCoreFiltersExtension'], loader=self.loader, undefined=StrictUndefined) + self.template_vars = {} + self.filename = '