Merge pull request #1 from stempler/feature/encyrption

first version of support for encrypted configuration files

README.md

@@ -45,7 +45,8 @@ root
 │  │  │     └─ Dockerfile
 │  │  └──config
 │  │     ├─ config1.yml
-│  │     └─ config2.env
+│  │     ├─ config2.vault.yml
+│  │     └─ config3.env
 │  │
 │  └──stack2
 │     ├─ swarm-composer.yml
@@ -94,6 +95,27 @@ YAML configurations are accessible via their property path (segments separated s
 
 Restrictions for variable evaluation in configuration files: Simple value insertions/replacements work, for conditions only boolean variables are supported right now. 
 
+#### Secret variables
+
+Sensible information like passwords can be stored in encrypted configuration files.
+These files then also for instance can be added to version control.
+
+For encrypted configuration files right now only the YAML format is supported, variables must be string values.
+
+To create an encrypted configuration file, first create its plain counterpart in the setup folder.
+The file names of the plain configuration files should end with `.secret.yml`.
+
+You also need to provide the password to use for the encryption.
+It can be provided as Gradle property, either for all setups (`vault_password`) or for individual setups (`vault_password_<setup>`).
+
+To encrypt the configuration file, run the encryption task for the respective setup (e.g. `./gradlew encrypt-<setup>`).
+Encrypted vault files have a file name that ends with `.vault.yml`.
+
+Note that when accessing the setup configuration, the plain files are recreated.
+If you want to remove them after a task, also add the `purgeSecrets` task.
+
+If you want to edit a vault file, you can either add encrpyted entries there, or simply decrypt the file with the task `decrypt-<setup>` and encrypt it after you completed your changes.
+
 #### Reserved variable names
 
 Some variables are provided by swarm-composer and will override any variables you define with the same name:

build.gradle

@@ -12,6 +12,9 @@ version = '1.0.0-SNAPSHOT'
 
 repositories {
   jcenter()
+  maven {
+    url 'https://jitpack.io'
+  }
 }
 
 dependencies{
@@ -29,6 +32,11 @@ dependencies{
 
   // Docker plugin
   compile 'com.bmuschko:gradle-docker-plugin:3.1.0'
+
+  // Encryption library
+  compile 'com.github.rockaport:alice:0.8.0', {
+    exclude group: 'org.codehaus.groovy', module: 'groovy-all'
+  }
 
   // testing
   testCompile 'junit:junit:4.12'

src/main/groovy/to/wetransform/gradle/swarm/SwarmComposerPlugin.groovy

@@ -17,13 +17,20 @@ import com.bmuschko.gradle.docker.tasks.image.DockerPushImage;
 
 import to.wetransform.gradle.swarm.actions.assemble.template.TemplateAssembler;
 import to.wetransform.gradle.swarm.config.ConfigHelper
-import to.wetransform.gradle.swarm.config.SetupConfiguration;
+import to.wetransform.gradle.swarm.config.SetupConfiguration
+import to.wetransform.gradle.swarm.crypt.ConfigCryptor
+import to.wetransform.gradle.swarm.crypt.SimpleConfigCryptor;
+import to.wetransform.gradle.swarm.crypt.alice.AliceCryptor;
 import to.wetransform.gradle.swarm.tasks.Assemble;
 
 class SwarmComposerPlugin implements Plugin<Project> {
 
   private static final Map<String, Object> DEFAULT_SC_CONFIG = [:]
 
+  private static final String PLAIN_FILE_IDENTIFIER = 'secret'
+
+  private static final String ENCRYPTED_FILE_IDENTIFIER = 'vault'
+
   private final def groovyEngine = new groovy.text.SimpleTemplateEngine()
 
   void apply(Project project) {
@@ -134,7 +141,8 @@ class SwarmComposerPlugin implements Plugin<Project> {
                 setupName: setup,
                 configFiles: configFiles,
                 settings: scConfig,
-                builds: stackBuilds.asImmutable())
+                builds: stackBuilds.asImmutable(),
+                setupDir: setupDir)
 
               configureSetup(project, sc)
             }
@@ -161,7 +169,8 @@ class SwarmComposerPlugin implements Plugin<Project> {
               setupName: 'default',
               configFiles: configFiles,
               settings: scConfig,
-              builds: stackBuilds.asImmutable())
+              builds: stackBuilds.asImmutable(),
+              setupDir: null)
 
             configureSetup(project, sc)
           }
@@ -220,7 +229,20 @@ class SwarmComposerPlugin implements Plugin<Project> {
       includes: includes,
       excludes: ['swarm-composer.yml'])
 
-    setupConfig.asCollection()
+    setupConfig.asCollection().collect { file ->
+      def name = file.name
+
+      // for secret files use plain counterpart
+      if (name.contains(".${ENCRYPTED_FILE_IDENTIFIER}.")) {
+        def plain = name.replaceAll("\\.${ENCRYPTED_FILE_IDENTIFIER}\\.",
+          ".${PLAIN_FILE_IDENTIFIER}.")
+        def neighbor = new File(file.parentFile, plain)
+        neighbor
+      }
+      else {
+        file
+      }
+    }.unique()
   }
 
   void configureSetup(Project project, final SetupConfiguration sc) {
@@ -234,6 +256,131 @@ class SwarmComposerPlugin implements Plugin<Project> {
 //      sc.config
 //    }
 
+    def vaultGroup = 'Configuration vault'
+
+    def purgeSecretsName = 'purgeSecrets'
+    def purgeSecretsTask = project.tasks.findByPath(purgeSecretsName)
+    if (!purgeSecretsTask) {
+      purgeSecretsTask = project.task(purgeSecretsName) {
+        group = vaultGroup
+        description = 'Delete all plain text secret files'
+      }
+    }
+
+    def decryptTask
+    if (sc.setupDir) {
+      // encryption / decryption tasks
+
+      // get password
+      def password = project.findProperty("vault_password_${sc.setupName}")
+      if (!password) {
+        password = project.findProperty("vault_password")
+      }
+
+      if (password) {
+        def encryptName = "encrypt-${sc.setupName}"
+        if (!project.tasks.findByPath(encryptName)) {
+          def encryptTask = project.task(encryptName) {
+            group = vaultGroup
+            description = "Create encrypted vault files from plain text secret files for setup ${sc.setupName}"
+          }.doFirst {
+            ConfigCryptor cryptor = new SimpleConfigCryptor(new AliceCryptor())
+
+            def files = project.fileTree(
+              dir: sc.setupDir,
+              includes: ["*.${PLAIN_FILE_IDENTIFIER}.*"]).asCollection()
+
+            files.each { plainFile ->
+              def name = plainFile.name.replaceAll("\\.${PLAIN_FILE_IDENTIFIER}\\.", ".${ENCRYPTED_FILE_IDENTIFIER}.")
+              def secretFile = new File(plainFile.parentFile, name)
+
+              /*
+               * XXX instead encrypt whole file?
+               *
+               * Advantages:
+               * - structure and comments preserved exactly
+               * - independent of file format
+               * Disadvantages:
+               * - not transparent which settings were changed in the encrypted file
+               *
+               * Both the file and the current implementation would allow handling
+               * encrypted configuration in memory without creating plain files.
+               * What stands in the way there is the fact that extended setups
+               * may have a different password protecting it.
+               */
+
+              // read, encrypt (with reference), write
+              //XXX only YAML supported right now
+              def config = ConfigHelper.loadYaml(plainFile)
+              def reference
+              if (secretFile.exists()) {
+                try {
+                  reference = ConfigHelper.loadYaml(secretFile)
+                } catch (e) {
+                  // ignore
+                }
+              }
+              config = cryptor.encrypt(config, password, reference)
+              ConfigHelper.saveYaml(config, secretFile)
+              // add comment to file
+              def now = new Date().toInstant().toString()
+              def comment = "# Encrypted configuration last updated on ${now}"
+              secretFile.text = comment + '\n' + secretFile.text
+            }
+          }
+        }
+
+        def decryptName = "decrypt-${sc.setupName}"
+        if (!project.tasks.findByPath(decryptName)) {
+          decryptTask = project.task(decryptName) {
+            group = vaultGroup
+            description = "Create plain text secret files from encrypted vault files for setup ${sc.setupName}"
+          }.doFirst {
+            ConfigCryptor cryptor = new SimpleConfigCryptor(new AliceCryptor())
+
+            def files = project.fileTree(
+              dir: sc.setupDir,
+              includes: ["*.${ENCRYPTED_FILE_IDENTIFIER}.*"]).asCollection()
+
+            files.each { secretFile ->
+              def name = secretFile.name.replaceAll("\\.${ENCRYPTED_FILE_IDENTIFIER}\\.",
+                ".${PLAIN_FILE_IDENTIFIER}.")
+              def plainFile = new File(secretFile.parentFile, name)
+
+              // read, decrypt, write
+              //XXX only YAML supported right now
+              def config = ConfigHelper.loadYaml(secretFile)
+              config = cryptor.decrypt(config, password)
+              ConfigHelper.saveYaml(config, plainFile)
+              // add comment to file
+              def now = new Date().toInstant().toString()
+              def comment = "# Decrypted configuration last updated on ${now}\n" +
+                '# DO NOT ADD TO VERSION CONTROL'
+              plainFile.text = comment + '\n' + plainFile.text
+            }
+          }
+        }
+
+        // purge task
+        def purgeName = "purgeSecrets-${sc.setupName}"
+        if (!project.tasks.findByPath(purgeName)) {
+          def purgeTask = project.task(purgeName) {
+            group = vaultGroup
+            description = "Delete all plain text secret files for setup ${sc.setupName}"
+          }.doLast {
+            project.fileTree(dir: sc.setupDir,
+                includes: ["*.${PLAIN_FILE_IDENTIFIER}.*"]).each { File file ->
+              file.delete()
+            }
+          }
+          purgeSecretsTask.dependsOn(purgeTask)
+        }
+
+      }
+    }
+
+
+    // assemble description
     def desc = "Generates compose file for stack ${sc.stackName} with setup ${sc.setupName}"
     def customDesc = sc.settings?.description?.trim()
     if (customDesc) {
@@ -307,6 +454,11 @@ $run"""
 
     setupPrepareTasks(project, task, sc)
 
+    // make sure decrypt task runs as part of preparation
+    if (decryptTask) {
+      project.tasks."prepareSetup-${sc.setupName}".dependsOn(decryptTask)
+    }
+
     // configure Docker image build tasks
     configureBuilds(project, sc, task)
 
@@ -480,13 +632,15 @@ $run"""
     }
   }
 
-  private void ensureTask(String name, String groupName, String descr, Project project) {
-    if (project.tasks.findByPath(name) == null) {
-      project.task(name) {
+  private Task ensureTask(String name, String groupName, String descr, Project project) {
+    def task = project.tasks.findByPath(name)
+    if (task == null) {
+      task = project.task(name) {
         group groupName
         description descr
       }
     }
+    task
   }
 
   private void setupPrepareTasks(Project project, Task task, SetupConfiguration sc, String build = null) {

src/main/groovy/to/wetransform/gradle/swarm/config/ConfigHelper.groovy

@@ -7,6 +7,7 @@ package to.wetransform.gradle.swarm.config
 
 import java.nio.charset.StandardCharsets
 
+import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.Yaml
 import org.yaml.snakeyaml.constructor.SafeConstructor
 
@@ -175,6 +176,24 @@ class ConfigHelper {
     yamlFile.withInputStream {
       result = yaml.load(it)
     }
+    result ?: [:]
+  }
+
+  /**
+   * Load a configuration from a YAML file.
+   *
+   * @param yamlFile the YAML file
+   * @return the loaded configuration map
+   */
+  static void saveYaml(Map config, File yamlFile) {
+    DumperOptions options = new DumperOptions()
+//    options.explicitStart = true
+    options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK)
+    Yaml yaml = new Yaml(options);
+    Map result
+    yamlFile.withWriter(StandardCharsets.UTF_8.name()) {
+      result = yaml.dump(config, it)
+    }
   }
 
 }

src/main/groovy/to/wetransform/gradle/swarm/config/SetupConfiguration.groovy

@@ -18,6 +18,8 @@ class SetupConfiguration {
 
   File stackFile
 
+  File setupDir
+
   String stackName
 
   String setupName

src/main/groovy/to/wetransform/gradle/swarm/crypt/ConfigCryptor.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2017 wetransform GmbH
+ * All rights reserved.
+ */
+
+package to.wetransform.gradle.swarm.crypt;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Interface for configuration encryption/decryption.
+ *
+ * @author Simon Templer
+ */
+public interface ConfigCryptor {
+
+  /**
+   * Encrypt the (string) settings in the given configuration and return the
+   * encrypted version of the configuration.
+   * The encryptor may mutate the given configuration and return it or create a new configuration.
+   *
+   * @param config the configuration
+   * @param password the password
+   * @param reference an already encrypted reference configuration (to reuse existing encryptions)
+   * @return the encrypted configuration
+   */
+  Map<String, Object> encrypt(Map<String, Object> config, String password, @Nullable Map<String, Object> reference) throws Exception;
+
+  /**
+   * Decrypt the (string) settings in the given configuration and return the
+   * decrypted version of the configuration.
+   * The decryptor may mutate the given configuration and return it or create a new configuration.
+   *
+   * @param config the configuration
+   * @param password the password
+   * @return the decrypted configuration
+   */
+  Map<String, Object> decrypt(Map<String, Object> config, String password) throws Exception;
+
+}

src/main/groovy/to/wetransform/gradle/swarm/crypt/Cryptor.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2017 wetransform GmbH
+ * All rights reserved.
+ */
+
+package to.wetransform.gradle.swarm.crypt;
+
+/**
+ * Encryption and decryption interface.
+ *
+ * @author Simon Templer
+ */
+public interface Cryptor {
+
+  String encrypt(String plain, String password) throws Exception;
+
+  String decrypt(String encrypted, String password) throws Exception;
+
+}