Merge pull request #28 from deepmind/compile-targets

Selective compilation of targets

README.md

@@ -79,37 +79,6 @@ Compiled minikube-es-2
 
 # Main concepts
 
-### Targets
-
-A target usually represents a single namespace in a kubernetes cluster and defines all components, scripts and documentation that will be generated for that target.
-
-Kapitan requires a target file to compile a target and its parameters. Example:
-
-```
-$ cat examples/targets/minikube-es/target.yml
----
-vars:
-  target: minikube-es
-  namespace: minikube-es
-compile:
-
-  - output_path: manifests
-    input_type: jsonnet
-    input_paths:
-      - targets/minikube-es/main.jsonnet
-    output_type: yaml
-
-  - output_path: scripts
-    input_type: jinja2
-    input_paths:
-      - scripts
-
-  - output_path: .
-    input_type: jinja2
-    input_paths:
-      - targets/minikube-es/docs/README.md
-```
-
 ### Components
 
 A component is an aplication that will be deployed to a kubernetes cluster. This includes all necessary kubernetes objects (StatefulSet, Services, ConfigMaps) defined in jsonnet.
@@ -153,6 +122,8 @@ parameters:
 
 #### Inventory Targets
 
+A target usually represents a single namespace in a kubernetes cluster and defines all components, scripts and documentation that will be generated for that target.
+
 Inside the inventory target files you can include classes and define new values or override any values inherited from the included classes.
 For example:
 
@@ -163,7 +134,30 @@ classes:
   - component.elasticsearch
 
 parameters:
-  namespace: minikube-es
+  target_name: minikube-es
+  kapitan:
+    vars:
+      target: ${target_name}
+      namespace: ${target_name}
+    compile:
+    - output_path: manifests
+      input_type: jsonnet
+      input_paths:
+        - components/elasticsearch/main.jsonnet
+      output_type: yaml
+    - output_path: scripts
+      input_type: jinja2
+      input_paths:
+        - scripts
+    - output_path: .
+      input_type: jinja2
+      input_paths:
+        - docs/elasticsearch/README.md
+
+    secrets:
+      recipients:
+        - dummy@recipient
+  namespace: ${target_name}
 
   elasticsearch:
     replicas: 2
@@ -254,7 +248,8 @@ optional arguments:
   --no-prune            do not prune jsonnet output
   --quiet               set quiet mode, only critical output
   --output-path PATH    set output path, default is "."
-  --target-path PATH    set target path, default is "./targets"
+  --targets TARGETS [TARGETS ...], -t TARGETS [TARGETS ...]
+                        targets to compile, default is all
   --parallelism INT, -p INT
                         Number of concurrent compile processes, default is 4
   --secrets-path SECRETS_PATH
@@ -328,6 +323,7 @@ classes:
   - cluster.common
   - cluster.minikube
   - component.elasticsearch
+environment: base
 parameters:
   cluster:
     id: minikube
@@ -443,5 +439,5 @@ We believe that these approaches can be blended in a powerful new way, glued tog
 
 # Related projects
 
-* [sublime-jsonnet-syntax](https://github.com/gburiola/sublime-jsonnet-syntax) - Jsonnet syntax highlighting for sublime text Edit
-
+* [sublime-jsonnet-syntax](https://github.com/gburiola/sublime-jsonnet-syntax) - Jsonnet syntax highlighting for Sublime Text
+* [language-jsonnet](https://github.com/google/language-jsonnet) - Jsonnet syntax highlighting for Atom

kapitan/cli.py

@@ -67,9 +67,8 @@ def main():
     compile_parser.add_argument('--output-path', type=str, default='.',
                                 metavar='PATH',
                                 help='set output path, default is "."')
-    compile_parser.add_argument('--target-path', type=str, default='targets',
-                                metavar='PATH',
-                                help='set target path, default is "./targets"')
+    compile_parser.add_argument('--targets', '-t', help='targets to compile, default is all',
+                                type=str, nargs='+', default=[], metavar='TARGET')
     compile_parser.add_argument('--parallelism', '-p', type=int,
                                 default=4, metavar='INT',
                                 help='Number of concurrent compile processes, default is 4')
@@ -154,13 +153,11 @@ def main():
             logging.basicConfig(level=logging.INFO, format="%(message)s")
         search_path = os.path.abspath(args.search_path)
         gpg_obj = secret_gpg_backend()
-        if args.target_path:
-            compile_targets(args.target_path, args.inventory_path, search_path,
-                            args.output_path, args.parallelism,
-                            prune=(not args.no_prune), secrets_path=args.secrets_path,
-                            secrets_reveal=args.reveal, gpg_obj=gpg_obj)
-        else:
-            logger.error("Error: Nothing to compile")
+
+        compile_targets(args.inventory_path, search_path, args.output_path,
+                        args.parallelism, args.targets,
+                        prune=(not args.no_prune), secrets_path=args.secrets_path,
+                        secrets_reveal=args.reveal, gpg_obj=gpg_obj)
 
     elif cmd == 'inventory':
         try:

kapitan/targets.py

@@ -40,30 +40,44 @@
 logger = logging.getLogger(__name__)
 
 
-def compile_targets(target_path, inventory_path, search_path, output_path, parallel, **kwargs):
+def compile_targets(inventory_path, search_path, output_path, parallel, targets, **kwargs):
     """
-    Searches and loads target files in target_path and runs compile_target_file() on a
+    Searches and loads target files, and runs compile_target_file() on a
     multiprocessing pool with parallel number of processes.
     kwargs are passed to compile_target()
     """
+    target_path = "targets"
     # temp_path will hold compiled items
     temp_path = tempfile.mkdtemp(suffix='.kapitan')
     pool = multiprocessing.Pool(parallel)
     # append "compiled" to output_path so we can safely overwrite it
     compile_path = os.path.join(output_path, "compiled")
     worker = partial(compile_target, search_path=search_path, compile_path=temp_path, **kwargs)
-    target_objs = load_target_files(target_path)
-    target_objs.extend(load_target_inventory(inventory_path))
+
+    target_objs = load_target_inventory(inventory_path, targets)
+
     try:
         if target_objs == []:
             logger.error("Error: no targets found")
             raise KapitanError("Error: no targets found")
         pool.map(worker, target_objs)
+
         if os.path.exists(compile_path):
-            shutil.rmtree(compile_path)
-        # on success, copy temp_path into compile_path
-        shutil.copytree(temp_path, compile_path)
-        logger.debug("Copied %s into %s", temp_path, compile_path)
+            # if '-t' is set on compile, only override selected targets
+            if targets:
+                for target in targets:
+                    compile_path_target = os.path.join(compile_path, target)
+                    temp_path_target = os.path.join(temp_path, target)
+
+                    shutil.rmtree(compile_path_target)
+                    shutil.copytree(temp_path_target, compile_path_target)
+                    logger.debug("Copied %s into %s", temp_path_target, compile_path_target)
+            # otherwise override all targets
+            else:
+                shutil.rmtree(compile_path)
+                shutil.copytree(temp_path, compile_path)
+                logger.debug("Copied %s into %s", temp_path, compile_path)
+
     except Exception as e:
         # if compile worker fails, terminate immediately
         pool.terminate()
@@ -79,32 +93,17 @@ def compile_targets(target_path, inventory_path, search_path, output_path, paral
         logger.debug("Removed %s", temp_path)
 
 
-def search_target_files(target_path):
-    """
-    Searches for any target.json or target.yml filenames in target_path
-    Returns a list of target objs
-    """
-    target_files = []
-    for root, _, files in os.walk(target_path):
-        for f in files:
-            if f in ('target.json', 'target.yml'):
-                full_path = os.path.join(root, f)
-                logger.debug('search_target_files: found %s', full_path)
-                target_files.append(full_path)
-    return target_files
-
-
-def load_target_files(target_path):
-    "return a list of target objects from target_files"
-    target_files = search_target_files(target_path)
-    return map(load_target_file, target_files)
-
-
-def load_target_inventory(inventory_path):
+def load_target_inventory(inventory_path, targets):
     "retuns a list of target objects from the inventory"
     target_objs = []
     inv = inventory_reclass(inventory_path)
-    for target_name in inv['nodes']:
+
+    targets_list = inv['nodes']
+    # if '-t' is set on compile, only loop through selected targets
+    if targets:
+        targets_list = targets
+
+    for target_name in targets_list:
         try:
             target_obj = inv['nodes'][target_name]['parameters']['kapitan']
             valid_target_obj(target_obj)