Merge pull request #14 from rundeck-plugins/upgrading-sdk

upgrading vault SDK to version 4.

build.gradle

@@ -51,7 +51,7 @@ dependencies {
     // add any third-party jar dependencies you wish to include in the plugin
     // using the `pluginLibs` configuration as shown here:
 
-    pluginLibs group: 'com.bettercloud', name: 'vault-java-driver', version: '3.1.0', ext: 'jar'
+    pluginLibs group: 'com.bettercloud', name: 'vault-java-driver', version: '4.0.0', ext: 'jar'
 
 
     //the compile dependency won't add the rundeck-core jar to the plugin contents

docker/.env

@@ -0,0 +1,4 @@
+VAULT_TOKEN=thisisatoken123.
+mem_limit=1200000000
+VAULT_IMAGE=vault:latest
+RUNDECK_IMAGE=rundeck/rundeck:SNAPSHOT

docker/Makefile

@@ -0,0 +1,25 @@
+all:
+
+#env vars
+DOCKER_COMPOSE_SPEC="docker-compose.yaml"
+build:
+	set -e
+	# re-build docker env
+	docker-compose -f $(DOCKER_COMPOSE_SPEC) build
+
+start:
+	# run docker
+	docker-compose -f $(DOCKER_COMPOSE_SPEC) up -d
+
+clean:
+	# clean up docker env
+	docker-compose -f $(DOCKER_COMPOSE_SPEC) down --volumes --remove-orphans
+
+build_and_start:
+	set -e
+	# re-build docker env
+	docker-compose -f $(DOCKER_COMPOSE_SPEC) build
+	# clean up docker env
+	docker-compose -f $(DOCKER_COMPOSE_SPEC) down --volumes --remove-orphans
+	# run docker
+	docker-compose -f $(DOCKER_COMPOSE_SPEC) up -d

docker/README.md

@@ -0,0 +1,29 @@
+## Rundeck-Vault integration example 
+
+This is a docker compose environment wih rundeck, mysql and vault
+
+### Requirements
+
+* Copy vault plugin to `docker/rundeck/plugins`
+
+### How to use it
+
+* Build
+
+```
+make build
+```
+
+* Start
+
+```
+make start
+```
+
+
+* Stop
+
+```
+make clean
+```
+

docker/docker-compose.yaml

@@ -0,0 +1,53 @@
+version: '2'
+
+services:
+  vault:
+    build:
+      context: vault
+      args:
+        IMAGE: ${VAULT_IMAGE}
+    ports:
+      - "8200:8200"
+    cap_add:
+      - IPC_LOCK
+    environment:
+      - VAULT_DEV_ROOT_TOKEN_ID=${VAULT_TOKEN}
+      - VAULT_DEV_LISTEN_ADDRESS=vault:8200
+      - VAULT_ADDR=http://vault:8200
+      - VAULT_TOKEN=${VAULT_TOKEN}
+  rundeck:
+    hostname: rundeck
+    build:
+      context: rundeck
+      args:
+        IMAGE: ${RUNDECK_IMAGE}
+    mem_limit: ${mem_limit}
+    command: -Dlogging.config=/home/rundeck/server/config/logback.groovy
+    links:
+    - vault
+    environment:
+    - RUNDECK_GRAILS_URL=http://localhost:4440
+    - RUNDECK_DATABASE_DRIVER=com.mysql.jdbc.Driver
+    - RUNDECK_DATABASE_USERNAME=rundeck
+    - RUNDECK_DATABASE_PASSWORD=rundeck
+    - RUNDECK_DATABASE_URL=jdbc:mysql://mysql/rundeck?autoReconnect=true&useSSL=false
+    - RUNDECK_STORAGE_PROVIDER_1_TYPE=vault-storage
+    - RUNDECK_STORAGE_PROVIDER_1_PATH=keys
+    - RUNDECK_STORAGE_PROVIDER_1_REMOVEPATHPREFIX=true
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_SECRETBACKEND=secret
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_PREFIX=app
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_ADDRESS=http://vault:8200
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_TOKEN=${VAULT_TOKEN}
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_BEHAVIOUR=vault
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_ENGINEVERSION=2
+    ports:
+    - "4440:4440"
+  mysql:
+    image: mysql:5.7
+    ports:
+      - "33060:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=root
+      - MYSQL_DATABASE=rundeck
+      - MYSQL_USER=rundeck
+      - MYSQL_PASSWORD=rundeck

docker/rundeck/Dockerfile

@@ -0,0 +1,38 @@
+ARG IMAGE
+FROM ${IMAGE}
+
+USER root
+
+RUN apt-get update && \
+    apt-get -y install apt-transport-https curl && \
+    apt-get -y install jq
+
+# add cli tool debian repo
+RUN echo "deb https://dl.bintray.com/rundeck/rundeck-deb /" | sudo tee -a /etc/apt/sources.list
+RUN curl "https://bintray.com/user/downloadSubjectPublicKey?username=bintray" > /tmp/bintray.gpg.key
+RUN apt-key add - < /tmp/bintray.gpg.key
+RUN apt-get -y update
+RUN apt-get -y install rundeck-cli
+
+# RUNDECK
+## RUNDECK setup env
+
+ENV USERNAME=rundeck \
+    USER=rundeck \
+    HOME=/home/rundeck \
+    LOGNAME=$USERNAME \
+    TERM=xterm-256color
+
+
+WORKDIR $HOME
+USER rundeck
+
+COPY --chown=rundeck:root remco /etc/remco
+COPY --chown=rundeck:root logback.groovy /home/rundeck/server/config
+
+COPY --chown=rundeck:root ./plugins ./libext
+
+VOLUME ["/home/rundeck/server/data"]
+
+EXPOSE 4440
+ENTRYPOINT [ "docker-lib/entry.sh" ]

docker/rundeck/logback.groovy

@@ -0,0 +1,23 @@
+import org.rundeck.util.logback.TrueConsoleAppender
+import org.springframework.boot.logging.logback.ColorConverter
+import org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter
+
+import java.nio.charset.Charset
+
+conversionRule 'clr', ColorConverter
+conversionRule 'wex', WhitespaceThrowableProxyConverter
+appender('STDOUT', TrueConsoleAppender){
+encoder(PatternLayoutEncoder) {
+        charset = Charset.forName('UTF-8')
+
+        pattern =
+                '%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} ' + // Date
+                        '%clr(%5p) ' + // Log level
+                        '%clr(---){faint} %clr([%15.15t]){faint} ' + // Thread
+                        '%clr(%-40.40logger{39}){cyan} %clr(:){faint} ' + // Logger
+                        '%m%n%wex' // Message
+    }
+}
+
+
+logger 'vault-storage', DEBUG, ['STDOUT'], true

docker/rundeck/plugins/README.md

@@ -0,0 +1 @@
+Add custom plugins here

docker/rundeck/remco/resources.d/rundeck-config-storage.properties.toml

@@ -0,0 +1,4 @@
+[[template]]
+    src         = "${REMCO_TEMPLATE_DIR}/rundeck-config-storage.properties"
+    dst         = "${REMCO_TMP_DIR}/rundeck-config/rundeck-config-storage.properties"
+    mode        = "0644"

docker/rundeck/remco/templates/rundeck-config-storage.properties

@@ -0,0 +1,30 @@
+{% set providerBase = "/rundeck/storage/provider" %}
+
+{% macro storage_provider(provider) %}
+{%- set index = provider | base %}
+rundeck.storage.provider.{{index}}.type={% set type = printf("%s/type", provider) %}{{ getv(type, "db")}}
+rundeck.storage.provider.{{index}}.path={% set path = printf("%s/path", provider) %}{{ getv(path, "keys")}}
+rundeck.storage.provider.{{index}}.removePathPrefix={% set removepathprefix = printf("%s/removepathprefix", provider) %}{{ getv(removepathprefix, "false")}}
+
+rundeck.storage.provider.{{index}}.config.prefix={% set prefix = printf("%s/config/prefix", provider) %}{{ getv(prefix, "")}}
+rundeck.storage.provider.{{index}}.config.address={% set address = printf("%s/config/address", provider) %}{{ getv(address, "")}}
+rundeck.storage.provider.{{index}}.config.token={% set token = printf("%s/config/token", provider) %}{{ getv(token, "")}}
+rundeck.storage.provider.{{index}}.config.storageBehaviour={% set behaviour = printf("%s/config/behaviour", provider) %}{{ getv(behaviour, "rundeck")}}
+rundeck.storage.provider.{{index}}.config.secretBackend={% set secretbackend = printf("%s/config/secretbackend", provider) %}{{ getv(secretbackend, "secret")}}
+
+rundeck.storage.provider.{{index}}.config.maxRetries={% set maxRetries = printf("%s/config/maxRetries", provider) %}{{ getv(maxRetries, "3")}}
+rundeck.storage.provider.{{index}}.config.retryIntervalMilliseconds={% set retryInterval = printf("%s/config/retryInterval", provider) %}{{ getv(retryInterval, "100")}}
+rundeck.storage.provider.{{index}}.config.openTimeout={% set openTimeout = printf("%s/config/openTimeout", provider) %}{{ getv(openTimeout, "3")}}
+rundeck.storage.provider.{{index}}.config.readTimeout={% set readTimeout = printf("%s/config/readTimeout", provider) %}{{ getv(readTimeout, "5")}}
+
+rundeck.storage.provider.{{index}}.config.engineVersion={% set engineversion = printf("%s/config/engineversion", provider) %}{{ getv(engineversion, "1")}}
+
+{% endmacro %}
+
+
+{% for p in lsdir(providerBase) -%}
+{% set provider = printf("%s/%s", providerBase, p) -%}
+{{ storage_provider(provider) }}
+{%- endfor %}
+
+

docker/vault/Dockerfile

@@ -0,0 +1,9 @@
+ARG IMAGE
+FROM ${IMAGE}
+
+EXPOSE 8200
+COPY run.sh /usr/local/bin/run.sh
+RUN apk --no-cache add curl
+RUN apk --no-cache add jq
+
+CMD ["run.sh"]

docker/vault/run.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env  sh
+
+#start vault
+
+vault server -config=/vault/config -dev & > log.out
+
+version=$(curl -s http://vault:8200/v1/sys/health |jq -r  .version)
+
+    echo "************ creating test keys"
+
+if (( $version > 1 )); then
+    echo "Vault 1.x"
+    vault kv put secret/app/simple.secret foo=world
+    vault kv put secret/app/multiples name=admin password=admin server=rundeck
+    vault kv put secret/app/folder/another.secret test=hello
+    vault kv put secret/app/folder/multiple2 name=admin password=admin server=rundeck
+
+else
+    echo "Vault 0.x"
+
+    vault write secret/app/simple.secret foo=world
+    vault write secret/app/multiples name=admin password=admin server=rundeck
+    vault write secret/app/folder/another.secret test=hello
+    vault write secret/app/folder/multiple2 name=admin password=admin server=rundeck
+
+
+    echo "************ end"
+
+fi
+sleep 10
+
+
+
+tail -f log.out

run-docker-vault-tests.sh

@@ -49,6 +49,7 @@ run_tests(){
 
 	bash $DIR/test-vault.sh
 	bash $DIR/test-existing-vault.sh
+    bash $DIR/test-existing-vault1.x.sh
 }
 run_docker_test(){
 	local FARGS=("$@")
@@ -63,4 +64,4 @@ main() {
     check_args
     run_docker_test $DOCKER_DIR
 }
-main
+main

src/main/java/io/github/valfadeev/rundeck/plugin/vault/ConfigOptions.java

@@ -24,4 +24,6 @@ class ConfigOptions {
     static final String VAULT_APPROLE_SECRET_ID = "approleSecretId";
     static final String VAULT_SECRET_BACKEND = "secretBackend";
     static final String VAULT_STORAGE_BEHAVIOUR = "storageBehaviour";
+    static final String VAULT_ENGINE_VERSION = "engineVersion";
+
 }

src/main/java/io/github/valfadeev/rundeck/plugin/vault/DescriptionProvider.java

@@ -4,11 +4,18 @@
 import com.dtolabs.rundeck.plugins.util.DescriptionBuilder;
 import com.dtolabs.rundeck.plugins.util.PropertyBuilder;
 
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
 import static io.github.valfadeev.rundeck.plugin.vault.SupportedAuthBackends.*;
 import static io.github.valfadeev.rundeck.plugin.vault.ConfigOptions.*;
 
 
 class DescriptionProvider {
+
+
+
     static Description getDescription() {
         return DescriptionBuilder.builder()
                 .name("vault-storage")
@@ -153,6 +160,13 @@ static Description getDescription() {
                         .description("Use the default Rundeck Behaviour for key storage (with rundeck headers) or use just the key/value behaviour from vault. Options are: rundeck, vault")
                         .defaultValue("rundeck")
                  )
+                .property(PropertyBuilder.builder()
+                        .select(VAULT_ENGINE_VERSION)
+                        .title("Vault Engine Version")
+                        .description("Key/Value Secret Engine Config")
+                        .values(Arrays.asList("1","2"))
+                        .defaultValue("1")
+                )
                 .build();
     }
 }

src/main/java/io/github/valfadeev/rundeck/plugin/vault/VaultClientProvider.java

@@ -24,13 +24,14 @@ class VaultClientProvider {
     Vault getVaultClient() throws ConfigurationException {
         final Integer vaultMaxRetries = Integer.parseInt(configuration.getProperty(VAULT_MAX_RETRIES));
         final Integer vaultRetryIntervalMilliseconds = Integer.parseInt(configuration.getProperty(VAULT_RETRY_INTERVAL_MILLISECONDS));
+        final Integer vaultEngineVersion = Integer.parseInt(configuration.getProperty(VAULT_ENGINE_VERSION));
 
         VaultConfig vaultConfig = getVaultConfig();
 
         try {
             String authToken = getVaultAuthToken();
             vaultConfig.token(authToken).build();
-            return new Vault(vaultConfig)
+            return new Vault(vaultConfig, vaultEngineVersion)
                     .withRetries(vaultMaxRetries,
                             vaultRetryIntervalMilliseconds);
         } catch (VaultException e) {

test/docker/docker-compose-existing-vault.yml

@@ -2,7 +2,10 @@ version: '2'
 
 services:
   vault:
-    build: dockers/vault
+    build:
+      context: dockers/vault
+      args:
+        IMAGE: vault:0.9.6
     ports:
       - "8200"
     cap_add:

test/docker/docker-compose-existing-vault1.x.yml

@@ -0,0 +1,43 @@
+version: '2'
+
+services:
+  vault:
+    build:
+      context: dockers/vault
+      args:
+        IMAGE: vault:1.1.3
+    ports:
+      - "8200"
+    cap_add:
+      - IPC_LOCK
+    environment:
+      - VAULT_DEV_ROOT_TOKEN_ID=${VAULT_TOKEN}
+      - VAULT_DEV_LISTEN_ADDRESS=vault:8200
+      - VAULT_ADDR=http://vault:8200
+      - VAULT_TOKEN=${VAULT_TOKEN}
+
+  rundeck1:
+    hostname: rundeck1
+    build:
+      context: dockers/rundeckvault/
+    links:
+    - vault
+    environment:
+    - RUNDECK_GRAILS_URL=http://localhost:4440
+    - RUNDECK_NODE=rundeck1
+    - SETUP_TEST_PROJECT=vaulttest
+    - RUNDECK_STORAGE_PROVIDER_1_TYPE=vault-storage
+    - RUNDECK_STORAGE_PROVIDER_1_PATH=keys
+    - RUNDECK_STORAGE_PROVIDER_1_REMOVEPATHPREFIX=false
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_SECRETBACKEND=secret
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_PREFIX=rundeck
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_ADDRESS=http://vault:8200
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_TOKEN=${VAULT_TOKEN}
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_BEHAVIOUR=vault
+    - RUNDECK_STORAGE_PROVIDER_1_CONFIG_ENGINEVERSION=2
+    - VAULT_TOKEN=${VAULT_TOKEN}
+    ports:
+    - "4440:4440"
+volumes:
+  logs:
+  resources: