diff --git a/README.md b/README.md index e64c30f..e20f385 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ root │ │ │ └─ Dockerfile │ │ └──config │ │ ├─ config1.yml -│ │ └─ config2.env +│ │ ├─ config2.vault.yml +│ │ └─ config3.env │ │ │ └──stack2 │ ├─ swarm-composer.yml @@ -94,6 +95,27 @@ YAML configurations are accessible via their property path (segments separated s Restrictions for variable evaluation in configuration files: Simple value insertions/replacements work, for conditions only boolean variables are supported right now. +#### Secret variables + +Sensible information like passwords can be stored in encrypted configuration files. +These files then also for instance can be added to version control. + +For encrypted configuration files right now only the YAML format is supported, variables must be string values. + +To create an encrypted configuration file, first create its plain counterpart in the setup folder. +The file names of the plain configuration files should end with `.secret.yml`. + +You also need to provide the password to use for the encryption. +It can be provided as Gradle property, either for all setups (`vault_password`) or for individual setups (`vault_password_`). + +To encrypt the configuration file, run the encryption task for the respective setup (e.g. `./gradlew encrypt-`). +Encrypted vault files have a file name that ends with `.vault.yml`. + +Note that when accessing the setup configuration, the plain files are recreated. +If you want to remove them after a task, also add the `purgeSecrets` task. + +If you want to edit a vault file, you can either add encrpyted entries there, or simply decrypt the file with the task `decrypt-` and encrypt it after you completed your changes. + #### Reserved variable names Some variables are provided by swarm-composer and will override any variables you define with the same name: diff --git a/build.gradle b/build.gradle index 8ee0373..494705a 100644 --- a/build.gradle +++ b/build.gradle @@ -12,6 +12,9 @@ version = '1.0.0-SNAPSHOT' repositories { jcenter() + maven { + url 'https://jitpack.io' + } } dependencies{ @@ -29,6 +32,11 @@ dependencies{ // Docker plugin compile 'com.bmuschko:gradle-docker-plugin:3.1.0' + + // Encryption library + compile 'com.github.rockaport:alice:0.8.0', { + exclude group: 'org.codehaus.groovy', module: 'groovy-all' + } // testing testCompile 'junit:junit:4.12' diff --git a/src/main/groovy/to/wetransform/gradle/swarm/SwarmComposerPlugin.groovy b/src/main/groovy/to/wetransform/gradle/swarm/SwarmComposerPlugin.groovy index aa3d8c9..8edb8e8 100644 --- a/src/main/groovy/to/wetransform/gradle/swarm/SwarmComposerPlugin.groovy +++ b/src/main/groovy/to/wetransform/gradle/swarm/SwarmComposerPlugin.groovy @@ -17,13 +17,20 @@ import com.bmuschko.gradle.docker.tasks.image.DockerPushImage; import to.wetransform.gradle.swarm.actions.assemble.template.TemplateAssembler; import to.wetransform.gradle.swarm.config.ConfigHelper -import to.wetransform.gradle.swarm.config.SetupConfiguration; +import to.wetransform.gradle.swarm.config.SetupConfiguration +import to.wetransform.gradle.swarm.crypt.ConfigCryptor +import to.wetransform.gradle.swarm.crypt.SimpleConfigCryptor; +import to.wetransform.gradle.swarm.crypt.alice.AliceCryptor; import to.wetransform.gradle.swarm.tasks.Assemble; class SwarmComposerPlugin implements Plugin { private static final Map DEFAULT_SC_CONFIG = [:] + private static final String PLAIN_FILE_IDENTIFIER = 'secret' + + private static final String ENCRYPTED_FILE_IDENTIFIER = 'vault' + private final def groovyEngine = new groovy.text.SimpleTemplateEngine() void apply(Project project) { @@ -134,7 +141,8 @@ class SwarmComposerPlugin implements Plugin { setupName: setup, configFiles: configFiles, settings: scConfig, - builds: stackBuilds.asImmutable()) + builds: stackBuilds.asImmutable(), + setupDir: setupDir) configureSetup(project, sc) } @@ -161,7 +169,8 @@ class SwarmComposerPlugin implements Plugin { setupName: 'default', configFiles: configFiles, settings: scConfig, - builds: stackBuilds.asImmutable()) + builds: stackBuilds.asImmutable(), + setupDir: null) configureSetup(project, sc) } @@ -220,7 +229,20 @@ class SwarmComposerPlugin implements Plugin { includes: includes, excludes: ['swarm-composer.yml']) - setupConfig.asCollection() + setupConfig.asCollection().collect { file -> + def name = file.name + + // for secret files use plain counterpart + if (name.contains(".${ENCRYPTED_FILE_IDENTIFIER}.")) { + def plain = name.replaceAll("\\.${ENCRYPTED_FILE_IDENTIFIER}\\.", + ".${PLAIN_FILE_IDENTIFIER}.") + def neighbor = new File(file.parentFile, plain) + neighbor + } + else { + file + } + }.unique() } void configureSetup(Project project, final SetupConfiguration sc) { @@ -234,6 +256,131 @@ class SwarmComposerPlugin implements Plugin { // sc.config // } + def vaultGroup = 'Configuration vault' + + def purgeSecretsName = 'purgeSecrets' + def purgeSecretsTask = project.tasks.findByPath(purgeSecretsName) + if (!purgeSecretsTask) { + purgeSecretsTask = project.task(purgeSecretsName) { + group = vaultGroup + description = 'Delete all plain text secret files' + } + } + + def decryptTask + if (sc.setupDir) { + // encryption / decryption tasks + + // get password + def password = project.findProperty("vault_password_${sc.setupName}") + if (!password) { + password = project.findProperty("vault_password") + } + + if (password) { + def encryptName = "encrypt-${sc.setupName}" + if (!project.tasks.findByPath(encryptName)) { + def encryptTask = project.task(encryptName) { + group = vaultGroup + description = "Create encrypted vault files from plain text secret files for setup ${sc.setupName}" + }.doFirst { + ConfigCryptor cryptor = new SimpleConfigCryptor(new AliceCryptor()) + + def files = project.fileTree( + dir: sc.setupDir, + includes: ["*.${PLAIN_FILE_IDENTIFIER}.*"]).asCollection() + + files.each { plainFile -> + def name = plainFile.name.replaceAll("\\.${PLAIN_FILE_IDENTIFIER}\\.", ".${ENCRYPTED_FILE_IDENTIFIER}.") + def secretFile = new File(plainFile.parentFile, name) + + /* + * XXX instead encrypt whole file? + * + * Advantages: + * - structure and comments preserved exactly + * - independent of file format + * Disadvantages: + * - not transparent which settings were changed in the encrypted file + * + * Both the file and the current implementation would allow handling + * encrypted configuration in memory without creating plain files. + * What stands in the way there is the fact that extended setups + * may have a different password protecting it. + */ + + // read, encrypt (with reference), write + //XXX only YAML supported right now + def config = ConfigHelper.loadYaml(plainFile) + def reference + if (secretFile.exists()) { + try { + reference = ConfigHelper.loadYaml(secretFile) + } catch (e) { + // ignore + } + } + config = cryptor.encrypt(config, password, reference) + ConfigHelper.saveYaml(config, secretFile) + // add comment to file + def now = new Date().toInstant().toString() + def comment = "# Encrypted configuration last updated on ${now}" + secretFile.text = comment + '\n' + secretFile.text + } + } + } + + def decryptName = "decrypt-${sc.setupName}" + if (!project.tasks.findByPath(decryptName)) { + decryptTask = project.task(decryptName) { + group = vaultGroup + description = "Create plain text secret files from encrypted vault files for setup ${sc.setupName}" + }.doFirst { + ConfigCryptor cryptor = new SimpleConfigCryptor(new AliceCryptor()) + + def files = project.fileTree( + dir: sc.setupDir, + includes: ["*.${ENCRYPTED_FILE_IDENTIFIER}.*"]).asCollection() + + files.each { secretFile -> + def name = secretFile.name.replaceAll("\\.${ENCRYPTED_FILE_IDENTIFIER}\\.", + ".${PLAIN_FILE_IDENTIFIER}.") + def plainFile = new File(secretFile.parentFile, name) + + // read, decrypt, write + //XXX only YAML supported right now + def config = ConfigHelper.loadYaml(secretFile) + config = cryptor.decrypt(config, password) + ConfigHelper.saveYaml(config, plainFile) + // add comment to file + def now = new Date().toInstant().toString() + def comment = "# Decrypted configuration last updated on ${now}\n" + + '# DO NOT ADD TO VERSION CONTROL' + plainFile.text = comment + '\n' + plainFile.text + } + } + } + + // purge task + def purgeName = "purgeSecrets-${sc.setupName}" + if (!project.tasks.findByPath(purgeName)) { + def purgeTask = project.task(purgeName) { + group = vaultGroup + description = "Delete all plain text secret files for setup ${sc.setupName}" + }.doLast { + project.fileTree(dir: sc.setupDir, + includes: ["*.${PLAIN_FILE_IDENTIFIER}.*"]).each { File file -> + file.delete() + } + } + purgeSecretsTask.dependsOn(purgeTask) + } + + } + } + + + // assemble description def desc = "Generates compose file for stack ${sc.stackName} with setup ${sc.setupName}" def customDesc = sc.settings?.description?.trim() if (customDesc) { @@ -307,6 +454,11 @@ $run""" setupPrepareTasks(project, task, sc) + // make sure decrypt task runs as part of preparation + if (decryptTask) { + project.tasks."prepareSetup-${sc.setupName}".dependsOn(decryptTask) + } + // configure Docker image build tasks configureBuilds(project, sc, task) @@ -480,13 +632,15 @@ $run""" } } - private void ensureTask(String name, String groupName, String descr, Project project) { - if (project.tasks.findByPath(name) == null) { - project.task(name) { + private Task ensureTask(String name, String groupName, String descr, Project project) { + def task = project.tasks.findByPath(name) + if (task == null) { + task = project.task(name) { group groupName description descr } } + task } private void setupPrepareTasks(Project project, Task task, SetupConfiguration sc, String build = null) { diff --git a/src/main/groovy/to/wetransform/gradle/swarm/config/ConfigHelper.groovy b/src/main/groovy/to/wetransform/gradle/swarm/config/ConfigHelper.groovy index ce0c1d1..a0e4eec 100644 --- a/src/main/groovy/to/wetransform/gradle/swarm/config/ConfigHelper.groovy +++ b/src/main/groovy/to/wetransform/gradle/swarm/config/ConfigHelper.groovy @@ -7,6 +7,7 @@ package to.wetransform.gradle.swarm.config import java.nio.charset.StandardCharsets +import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml import org.yaml.snakeyaml.constructor.SafeConstructor @@ -175,6 +176,24 @@ class ConfigHelper { yamlFile.withInputStream { result = yaml.load(it) } + result ?: [:] + } + + /** + * Load a configuration from a YAML file. + * + * @param yamlFile the YAML file + * @return the loaded configuration map + */ + static void saveYaml(Map config, File yamlFile) { + DumperOptions options = new DumperOptions() +// options.explicitStart = true + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK) + Yaml yaml = new Yaml(options); + Map result + yamlFile.withWriter(StandardCharsets.UTF_8.name()) { + result = yaml.dump(config, it) + } } } diff --git a/src/main/groovy/to/wetransform/gradle/swarm/config/SetupConfiguration.groovy b/src/main/groovy/to/wetransform/gradle/swarm/config/SetupConfiguration.groovy index a471c43..7aee23a 100644 --- a/src/main/groovy/to/wetransform/gradle/swarm/config/SetupConfiguration.groovy +++ b/src/main/groovy/to/wetransform/gradle/swarm/config/SetupConfiguration.groovy @@ -18,6 +18,8 @@ class SetupConfiguration { File stackFile + File setupDir + String stackName String setupName diff --git a/src/main/groovy/to/wetransform/gradle/swarm/crypt/ConfigCryptor.java b/src/main/groovy/to/wetransform/gradle/swarm/crypt/ConfigCryptor.java new file mode 100644 index 0000000..fa46a70 --- /dev/null +++ b/src/main/groovy/to/wetransform/gradle/swarm/crypt/ConfigCryptor.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 wetransform GmbH + * All rights reserved. + */ + +package to.wetransform.gradle.swarm.crypt; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Interface for configuration encryption/decryption. + * + * @author Simon Templer + */ +public interface ConfigCryptor { + + /** + * Encrypt the (string) settings in the given configuration and return the + * encrypted version of the configuration. + * The encryptor may mutate the given configuration and return it or create a new configuration. + * + * @param config the configuration + * @param password the password + * @param reference an already encrypted reference configuration (to reuse existing encryptions) + * @return the encrypted configuration + */ + Map encrypt(Map config, String password, @Nullable Map reference) throws Exception; + + /** + * Decrypt the (string) settings in the given configuration and return the + * decrypted version of the configuration. + * The decryptor may mutate the given configuration and return it or create a new configuration. + * + * @param config the configuration + * @param password the password + * @return the decrypted configuration + */ + Map decrypt(Map config, String password) throws Exception; + +} diff --git a/src/main/groovy/to/wetransform/gradle/swarm/crypt/Cryptor.java b/src/main/groovy/to/wetransform/gradle/swarm/crypt/Cryptor.java new file mode 100644 index 0000000..8d99610 --- /dev/null +++ b/src/main/groovy/to/wetransform/gradle/swarm/crypt/Cryptor.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 wetransform GmbH + * All rights reserved. + */ + +package to.wetransform.gradle.swarm.crypt; + +/** + * Encryption and decryption interface. + * + * @author Simon Templer + */ +public interface Cryptor { + + String encrypt(String plain, String password) throws Exception; + + String decrypt(String encrypted, String password) throws Exception; + +} diff --git a/src/main/groovy/to/wetransform/gradle/swarm/crypt/SimpleConfigCryptor.groovy b/src/main/groovy/to/wetransform/gradle/swarm/crypt/SimpleConfigCryptor.groovy new file mode 100644 index 0000000..9a29478 --- /dev/null +++ b/src/main/groovy/to/wetransform/gradle/swarm/crypt/SimpleConfigCryptor.groovy @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2017 wetransform GmbH + * All rights reserved. + */ + +package to.wetransform.gradle.swarm.crypt + +import java.util.List; +import java.util.Map +import java.util.Set +import java.util.function.Function;;; + +/** + * Applies encryption/decryption to configuration. + * + * @author Simon Templer + */ +class SimpleConfigCryptor implements ConfigCryptor { + + private final Cryptor cryptor + + + SimpleConfigCryptor(Cryptor cryptor) { + super() + this.cryptor = cryptor + } + + @Override + Map encrypt(Map config, String password, Map reference) throws Exception { + apply(config) { value, path -> + def reuse + + if (reference) { + // look up path and check if value should be reused + def refValue = reference + for (int i = 0; i < path.size() && refValue; i++) { + refValue = refValue?."${path[i]}" + } + if (refValue) { + refValue = refValue.toString() + try { + def decrypted = cryptor.decrypt(refValue, password) + if (decrypted == value) { + reuse = refValue + } + } catch (Exception e) { + // ignore + } + } + } + + reuse ?: cryptor.encrypt(value, password) + } + config + } + + @Override + Map decrypt(Map config, String password) throws Exception { + apply(config) { value -> + cryptor.decrypt(value, password) + } + config + } + + private void apply(Map config, Closure crypt, List path = []) { + def keys = new LinkedHashSet(config.keySet()) + + keys.each { key -> + def value = config[key] + + if (value != null) { + def childPath = [] + childPath.addAll(path) + childPath.add(key) + + if (value instanceof List) { + //FIXME lists are not supported ATM + } + else if (value instanceof String || value instanceof GString) { + // evaluate value + + def newValue + if (crypt.maximumNumberOfParameters > 1) { + newValue = crypt(value.toString(), childPath) + } + else { + newValue = crypt(value.toString()) + } + + config.put(key, newValue) + } + // proceed to child maps + else if (value instanceof Map) { + apply(value, crypt, childPath) + } + } + } + } + +} diff --git a/src/main/groovy/to/wetransform/gradle/swarm/crypt/alice/AliceCryptor.groovy b/src/main/groovy/to/wetransform/gradle/swarm/crypt/alice/AliceCryptor.groovy new file mode 100644 index 0000000..23cce2a --- /dev/null +++ b/src/main/groovy/to/wetransform/gradle/swarm/crypt/alice/AliceCryptor.groovy @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 wetransform GmbH + * All rights reserved. + */ + +package to.wetransform.gradle.swarm.crypt.alice; + +import java.nio.charset.StandardCharsets; + +import com.rockaport.alice.Alice; +import com.rockaport.alice.AliceContext +import com.rockaport.alice.AliceContext.KeyLength; +import com.rockaport.alice.AliceContextBuilder; + +import to.wetransform.gradle.swarm.crypt.Cryptor; + +/** + * Cryptor based on Alice encryption library. + * + * @author Simon Templer + */ +public class AliceCryptor implements Cryptor { + + private final Alice alice + + public AliceCryptor() { + super() + + //TODO based on configuration? + + AliceContext aliceContext = new AliceContextBuilder() + .setAlgorithm(AliceContext.Algorithm.AES) + // on Oracle Java may require to install + // http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html + .setKeyLength(KeyLength.BITS_256) + .setMode(AliceContext.Mode.GCM) + .setIvLength(12) + .setGcmTagLength(AliceContext.GcmTagLength.BITS_128) + .build() + alice = new Alice(aliceContext) + } + + @Override + public String encrypt(String plain, String password) { + byte[] data = alice.encrypt(plain.getBytes(StandardCharsets.UTF_8), password.toCharArray()) + data.encodeBase64().toString() + } + + @Override + public String decrypt(String encrypted, String password) { + byte[] decoded = encrypted.decodeBase64() + byte[] decrypted = alice.decrypt(decoded, password.toCharArray()) + new String(decrypted, StandardCharsets.UTF_8) + } + +} diff --git a/src/test/groovy/to/wetransform/gradle/swarm/crypt/SimpleConfigCryptorTest.groovy b/src/test/groovy/to/wetransform/gradle/swarm/crypt/SimpleConfigCryptorTest.groovy new file mode 100644 index 0000000..8d6269c --- /dev/null +++ b/src/test/groovy/to/wetransform/gradle/swarm/crypt/SimpleConfigCryptorTest.groovy @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017 wetransform GmbH + * All rights reserved. + */ + +package to.wetransform.gradle.swarm.crypt + +import org.junit.Test + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper; +import to.wetransform.gradle.swarm.crypt.alice.AliceCryptor;; + +/** + * Tests for SimpleConfigCryptor + * + * @author Simon Templer + */ +class SimpleConfigCryptorTest { + + @Test + void testEncryptDecrypt() { + AliceCryptor alice = new AliceCryptor() + SimpleConfigCryptor c = new SimpleConfigCryptor(alice) + + def plain = [ + greeting: "Hello world", + more: [ + title: 'Title', + content: '''Lorem + +ipsum - or what?''' + ] + ] + def copy = new JsonSlurper().parseText(JsonOutput.toJson(plain)) + + assert plain == copy + + String password = "Goodbye" + + def encrypted = c.encrypt(copy, password) + def copyEncrypted = new JsonSlurper().parseText(JsonOutput.toJson(encrypted)) + + assert encrypted == copyEncrypted + + println(encrypted.inspect()) + + def decrypted = c.decrypt(copyEncrypted, password) + + assert plain == decrypted + + assert encrypted != decrypted + } + +} diff --git a/src/test/groovy/to/wetransform/gradle/swarm/crypt/alice/AliceCryptorTest.groovy b/src/test/groovy/to/wetransform/gradle/swarm/crypt/alice/AliceCryptorTest.groovy new file mode 100644 index 0000000..81887e2 --- /dev/null +++ b/src/test/groovy/to/wetransform/gradle/swarm/crypt/alice/AliceCryptorTest.groovy @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2017 wetransform GmbH + * All rights reserved. + */ + +package to.wetransform.gradle.swarm.crypt.alice + +import org.junit.Test; + +/** + * Simple AliceCryptor test. + * + * @author Simon Templer + */ +class AliceCryptorTest { + + @Test + void testEncryptDecrypt() { + AliceCryptor c = new AliceCryptor() + + String plain = "Hello world" + String password = "Goodbye" + + String encrypted = c.encrypt(plain, password) + + String decrypted = c.decrypt(encrypted, password) + + assert plain == decrypted + + assert encrypted != decrypted + } + +}