diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d39cf08 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +build +.gradle +gradle \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..87a1bb8 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# jq-JSON mapper log filter plugin + +This plugin maps a JSON String output from a step to key/value context variables. +It uses the [jackson-jq](https://github.com/eiiches/jackson-jq) java library. + +## Build + +``` +gradle clean install +``` + +## Install + +``` +cp build/lib/jq-json-logfilter-X.X.X.jar $RDECKBASE/libext +``` + +## How to use + +- Add a Log filter to the workflow step what has a json output string + +![Add Filter](examples/example1.png) + +More information about the available jq Filters [here](https://github.com/eiiches/jackson-jq#implementation-status-and-current-limitations) + + +- Use the context variables on the next steps + +![Add Filter](examples/example2.png) + +![Add Filter](examples/example3.png) + diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..f0a6ef8 --- /dev/null +++ b/build.gradle @@ -0,0 +1,91 @@ +plugins { + id 'groovy' + id 'java' + id 'pl.allegro.tech.build.axion-release' version '1.7.1' +} + +group 'com.rundeck.plugin' + +ext.rundeckVersion='3.0.6-20180917' +defaultTasks 'clean','build' +apply plugin: 'java' +apply plugin: 'idea' + +sourceCompatibility = 1.8 +ext.rundeckPluginVersion= '1.2' + +scmVersion { + ignoreUncommittedChanges = false + tag { + prefix = '' + versionSeparator = '' + def origDeserialize=deserialize + //apend .0 to satisfy semver if the tag version is only X.Y + deserialize = { config, position, tagName -> + def orig = origDeserialize(config, position, tagName) + if (orig.split('\\.').length < 3) { + orig += ".0" + } + orig + } + } +} +project.version = scmVersion.version + +/** + * Set this to a comma-separated list of full classnames of your implemented Rundeck + * plugins. + */ +ext.pluginClassNames='com.rundeck.plugins.logging.JsonLogFilter' + + +sourceCompatibility = 1.8 + +repositories { + mavenCentral() +} + +configurations{ + //declare custom pluginLibs configuration to include only libs for this plugin + pluginLibs + + //declare compile to extend from pluginLibs so it inherits the dependencies + compile{ + extendsFrom pluginLibs + } +} + + +dependencies { + compile 'org.codehaus.groovy:groovy-all:2.3.11' + + compile group: 'org.rundeck', name: 'rundeck-core', version: rundeckVersion + pluginLibs group: 'net.thisptr', name: 'jackson-jq', version: '0.0.9' + + + testCompile group: 'junit', name: 'junit', version: '4.12' + testCompile "org.spockframework:spock-core:0.7-groovy-2.0" +} + + +// task to copy plugin libs to output/lib dir +task copyToLib(type: Copy) { + into "$buildDir/output/lib" + from configurations.pluginLibs +} + + +jar { + from "$buildDir/output" + manifest { + def libList = configurations.pluginLibs.collect{'lib/'+it.name}.join(' ') + attributes 'Rundeck-Plugin-Classnames': pluginClassNames + attributes 'Rundeck-Plugin-File-Version': version + attributes 'Rundeck-Plugin-Version': rundeckPluginVersion, 'Rundeck-Plugin-Archive': 'true' + attributes 'Rundeck-Plugin-Libs': "${libList}" + attributes 'Main-Class': "io.github.valfadeev.rundeck.plugin.vault.VaultStoragePlugin" + attributes 'Class-Path': "${libList} lib/rundeck-core-${rundeckVersion}.jar" + } +} +//set jar task to depend on copyToLib +jar.dependsOn(copyToLib) \ No newline at end of file diff --git a/examples/example1.png b/examples/example1.png new file mode 100644 index 0000000..81e19c7 Binary files /dev/null and b/examples/example1.png differ diff --git a/examples/example2.png b/examples/example2.png new file mode 100644 index 0000000..ccc8d65 Binary files /dev/null and b/examples/example2.png differ diff --git a/examples/example3.png b/examples/example3.png new file mode 100644 index 0000000..54c4672 Binary files /dev/null and b/examples/example3.png differ diff --git a/examples/test-json-array.xml b/examples/test-json-array.xml new file mode 100644 index 0000000..648bc33 --- /dev/null +++ b/examples/test-json-array.xml @@ -0,0 +1,47 @@ + + + summary + + true + JSON + INFO + test-json-array + false + true + + + + + + .[] + true + data + + + + + + + + echo ${data.created_date} + + + + \ No newline at end of file diff --git a/examples/test-json-simple.xml b/examples/test-json-simple.xml new file mode 100644 index 0000000..255a529 --- /dev/null +++ b/examples/test-json-simple.xml @@ -0,0 +1,47 @@ + + + summary + + true + JSON + INFO + test-json-simple + false + true + + + + + + . + true + data + + + + + + + + echo ${data.result.created_date} + + + + \ No newline at end of file diff --git a/src/main/groovy/com/rundeck/plugins/logging/JsonLogFilter.groovy b/src/main/groovy/com/rundeck/plugins/logging/JsonLogFilter.groovy new file mode 100644 index 0000000..8f88fa7 --- /dev/null +++ b/src/main/groovy/com/rundeck/plugins/logging/JsonLogFilter.groovy @@ -0,0 +1,165 @@ +package com.rundeck.plugins.logging + +import com.dtolabs.rundeck.core.execution.workflow.OutputContext +import com.dtolabs.rundeck.core.logging.LogEventControl +import com.dtolabs.rundeck.core.logging.LogLevel +import com.dtolabs.rundeck.core.logging.PluginLoggingContext +import com.dtolabs.rundeck.core.plugins.Plugin +import com.dtolabs.rundeck.plugins.descriptions.PluginDescription +import com.dtolabs.rundeck.plugins.descriptions.PluginProperty +import com.dtolabs.rundeck.plugins.logging.LogFilterPlugin +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeType +import net.thisptr.jackson.jq.JsonQuery +import net.thisptr.jackson.jq.Scope + +@Plugin(name = JsonLogFilter.PROVIDER_NAME, service = 'LogFilter') +@PluginDescription(title = 'JSON jq key/value mapper', + description = 'Map a json logoutput on key/value to context data using jq filters') +class JsonLogFilter implements LogFilterPlugin{ + public static final String PROVIDER_NAME = 'json-mapper' + + @PluginProperty( + title = 'jq Filter', + description = '''Add a jq Filter''', + defaultValue=".", + required=true + + ) + String filter + + @PluginProperty( + title = 'Prefix', + description = '''(Optional) Result prefix''', + defaultValue="result" + ) + String prefix + + @PluginProperty( + title = 'Log Data', + description = '''If true, log the captured data''', + defaultValue = 'false' + ) + Boolean logData + + private StringBuffer buffer; + OutputContext outputContext + Map allData + private ObjectMapper mapper + + + @Override + void init(final PluginLoggingContext context) { + outputContext = context.getOutputContext() + buffer = new StringBuffer() + mapper = new ObjectMapper() + allData = [:] + + } + + @Override + void handleEvent(final PluginLoggingContext context, final LogEventControl event) { + if (event.eventType == 'log' && event.loglevel == LogLevel.NORMAL && event.message?.length() > 0) { + buffer.append(event.message) + } + + } + + @Override + void complete(final PluginLoggingContext context) { + + if(buffer.size()>0) { + //apply json transform + this.processJson(context) + + if (logData) { + context.log( + 2, + mapper.writeValueAsString(allData), + [ + 'content-data-type' : 'application/json', + 'content-meta:table-title': 'Key Value Data: Results' + ] + ) + } + } + } + + void processJson(final PluginLoggingContext context){ + + try{ + + Scope rootScope = Scope.newEmptyScope() + rootScope.loadFunctions(Scope.class.getClassLoader()) + JsonQuery q = JsonQuery.compile(filter) + JsonNode inData = mapper.readTree(buffer.toString()); + + List out = q.apply(rootScope, inData) + + out.each {it-> + //process object + if(it.getNodeType()==JsonNodeType.OBJECT){ + def map = it.fields() + for (Map.Entry entry : map) { + this.iterateJsonObject(entry) + } + }else{ + if(it.getNodeType()==JsonNodeType.ARRAY){ + this.iterateArray(it.elements()) + }else{ + allData.put(prefix, it.toString()) + } + } + } + + outputContext.addOutput("data",allData) + + }catch(Exception exception){ + context.log(0, "[error] cannot map the json output: " + exception.message) + } + } + + def iterateArray(def list){ + + Integer i=0 + list.each{it-> + if(it.getNodeType() == JsonNodeType.STRING){ + allData.put(prefix +"."+ i.toString(),it.textValue()) + }else{ + allData.put(prefix +"."+ i.toString(),it.toString()) + } + + i++ + } + } + + def iterateJsonObject(Map.Entry entry, String path=""){ + + def key = entry.getKey() + def value = entry.getValue() + def newPath + + if(!path){ + newPath=key + }else{ + newPath=path + "."+ key + } + + if(value.getNodeType()== JsonNodeType.OBJECT){ + for (Map.Entry subKey : entry.getValue().fields()) { + iterateJsonObject(subKey, newPath) + } + }else{ + def extractValue + if(value.getNodeType() == JsonNodeType.STRING){ + extractValue=value.textValue() + }else{ + extractValue = value.toString() + } + + allData.put(newPath,extractValue) + } + } + +} diff --git a/src/test/groovy/com/rundeck/plugins/logging/JsonLogFilterSpec.groovy b/src/test/groovy/com/rundeck/plugins/logging/JsonLogFilterSpec.groovy new file mode 100644 index 0000000..8e2f1d9 --- /dev/null +++ b/src/test/groovy/com/rundeck/plugins/logging/JsonLogFilterSpec.groovy @@ -0,0 +1,135 @@ +package com.rundeck.plugins.logging + +import com.dtolabs.rundeck.core.dispatcher.ContextView +import com.dtolabs.rundeck.core.execution.workflow.DataOutput +import com.dtolabs.rundeck.core.logging.LogEventControl +import com.dtolabs.rundeck.core.logging.LogLevel +import com.dtolabs.rundeck.core.logging.PluginLoggingContext +import spock.lang.Specification + +class JsonLogFilterSpec extends Specification { + + def "simple test"() { + given: + def plugin = new JsonLogFilter() + plugin.filter = filter + plugin.prefix = "result" + plugin.logData = dolog + def sharedoutput = new DataOutput(ContextView.global()) + def context = Mock(PluginLoggingContext) { + getOutputContext() >> sharedoutput + } + def events = [] + lines.each { line -> + events << Mock(LogEventControl) { + getMessage() >> line + getEventType() >> 'log' + getLoglevel() >> LogLevel.NORMAL + } + } + when: + plugin.init(context) + events.each { + plugin.handleEvent(context, it) + } + plugin.complete(context) + then: + + sharedoutput.getSharedContext().getData(ContextView.global())?.getData() == (expect ? ['data': expect] : null) + if (expect) { + if (dolog) { + 1 * context.log(2, _, _) + } else { + 0 * context.log(*_) + } + } + + + where: + dolog | filter | lines | expect + true | "." | ['{"test":"value"}'] | [test: 'value'] + true | ".| keys" | ['{"test":"value","something2":"value1"}'] | + [ + 'result.0' : 'something2', + 'result.1' : 'test' + ] + false | ".| length" | ['{"test":"value","something2":"value1"}'] | [result: '2'] + true | "." | ['{"test":"value","something2":"value1"}'] | + [ + 'test' : 'value', + 'something2' : 'value1' + ] + } + + def "array test"() { + given: + def filter = ".[]" + def dolog = true + def plugin = new JsonLogFilter() + plugin.filter = filter + plugin.prefix = "result" + plugin.logData = dolog + def sharedoutput = new DataOutput(ContextView.global()) + def context = Mock(PluginLoggingContext) { + getOutputContext() >> sharedoutput + } + def events = [] + + def lines = "{\n" + + " \"result\": {\n" + + " \"parent\": {\n" + + " \"link\": \"https://rundeck-test/api\",\n" + + " \"value\": \"123345\"\n" + + " },\n" + + " \"user\": \"rundeck\",\n" + + " \"created_by\": {\n" + + " \"link\": \"https://rundeck-test/user/1\",\n" + + " \"value\": \"123345\"\n" + + " },\n" + + " \"created_date\": \"2018-10-03 19:17:54\",\n" + + " \"status\": \"1\",\n" + + " \"closed_date\": \"2018-10-03 19:46:49\"\n" + + " }\n" + + "}" + + def expect = [ + 'parent.link' : 'https://rundeck-test/api', + 'parent.value' : '123345', + 'user' : 'rundeck', + 'created_by.link' : 'https://rundeck-test/user/1', + 'created_by.value' : '123345', + 'created_date' : '2018-10-03 19:17:54', + 'status' : '1', + 'closed_date' : '2018-10-03 19:46:49', + ] + + lines.each { line -> + events << Mock(LogEventControl) { + getMessage() >> line + getEventType() >> 'log' + getLoglevel() >> LogLevel.NORMAL + } + } + when: + plugin.init(context) + events.each { + plugin.handleEvent(context, it) + } + plugin.complete(context) + then: + + + + sharedoutput.getSharedContext().getData(ContextView.global())?.getData() == (expect ? ['data': expect] : null) + if (expect) { + if (dolog) { + 1 * context.log(2, _, _) + } else { + 0 * context.log(*_) + } + } + + + } + +}