From 981c1bdde31eb8951f61be1d5cfa37d2919fc364 Mon Sep 17 00:00:00 2001 From: Christian McHugh Date: Sat, 10 Jun 2017 22:38:12 +0100 Subject: [PATCH] Stepping (#95) * initial step support * make saltStep calls happen in thread * save output as variable in pipeline * Support resume with jid --- pom.xml | 4 +- src/main/java/com/waytta/Builds.java | 20 +- src/main/java/com/waytta/SaltAPIBuilder.java | 16 +- src/main/java/com/waytta/SaltAPIStep.java | 183 ++++++++++++++---- src/main/java/com/waytta/Utils.java | 2 - .../waytta/clientinterface/BasicClient.java | 97 +++++----- 6 files changed, 221 insertions(+), 101 deletions(-) diff --git a/pom.xml b/pom.xml index 0d7e9af..4ba17b4 100644 --- a/pom.xml +++ b/pom.xml @@ -118,8 +118,8 @@ org.jenkins-ci.plugins.workflow - workflow-basic-steps - 1.2 + workflow-step-api + 2.3 org.powermock diff --git a/src/main/java/com/waytta/Builds.java b/src/main/java/com/waytta/Builds.java index 788ce00..9f6e993 100644 --- a/src/main/java/com/waytta/Builds.java +++ b/src/main/java/com/waytta/Builds.java @@ -1,7 +1,6 @@ package com.waytta; import hudson.model.Run; -import hudson.model.Result; import hudson.model.TaskListener; import java.io.IOException; @@ -125,29 +124,34 @@ public static JSONArray returnData(JSONObject saltReturn, String netapi) { return returnArray; } - public static JSONArray runBlockingBuild(Launcher launcher, Run build, String myservername, - String token, JSONObject saltFunc, TaskListener listener, int pollTime, int minionTimeout, String netapi) + public static String getBlockingBuildJid(Launcher launcher, String myservername, + String token, JSONObject saltFunc, TaskListener listener) throws IOException, InterruptedException, SaltException { - JSONArray returnArray = new JSONArray(); - JSONObject httpResponse = new JSONObject(); - String jid = ""; + String jid = null; // Send request to /minion url. This will give back a jid which we // will need to poll and lookup for completion - httpResponse = launcher.getChannel().call(new HttpCallable(myservername + "/minions", saltFunc, token)); - returnArray = httpResponse.getJSONArray("return"); + JSONObject httpResponse = launcher.getChannel().call(new HttpCallable(myservername + "/minions", saltFunc, token)); + JSONArray returnArray = httpResponse.getJSONArray("return"); for (Object o : returnArray) { JSONObject line = (JSONObject) o; jid = line.getString("jid"); } // Print out success listener.getLogger().println("Running jid: " + jid); + return jid; + } + public static JSONArray checkBlockingBuild(Launcher launcher, String myservername, String token, + JSONObject saltFunc, TaskListener listener, int pollTime, int minionTimeout, String netapi, String jid) + throws IOException, InterruptedException, SaltException { // Request successfully sent. Now use jid to check if job complete int numMinions = 0; int numMinionsDone = 0; JSONArray minionsArray = new JSONArray(); JSONObject resultObject = new JSONObject(); JSONArray httpArray = new JSONArray(); + JSONArray returnArray = new JSONArray(); + JSONObject httpResponse = new JSONObject(); httpResponse = launcher.getChannel().call(new HttpCallable(myservername + "/jobs/" + jid, null, token)); httpArray = returnData(httpResponse, netapi); for (Object o : httpArray) { diff --git a/src/main/java/com/waytta/SaltAPIBuilder.java b/src/main/java/com/waytta/SaltAPIBuilder.java index eceb129..92c80da 100644 --- a/src/main/java/com/waytta/SaltAPIBuilder.java +++ b/src/main/java/com/waytta/SaltAPIBuilder.java @@ -1,6 +1,7 @@ package com.waytta; import java.io.IOException; +import java.io.Serializable; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; @@ -61,7 +62,9 @@ import net.sf.json.JSONObject; import net.sf.json.util.JSONUtils; -public class SaltAPIBuilder extends Builder implements SimpleBuildStep { +public class SaltAPIBuilder extends Builder implements SimpleBuildStep, Serializable { + private static final long serialVersionUID = 1L; + private static final Logger LOGGER = Logger.getLogger("com.waytta.saltstack"); private String servername; @@ -171,7 +174,6 @@ public String getTag() { return clientInterface.getTag(); } - @Override public void perform(Run build, FilePath workspace, Launcher launcher, TaskListener listener) throws InterruptedException, IOException { @@ -259,6 +261,13 @@ public void perform(Run build, FilePath workspace, Launcher launcher, Task } } + public String getJID(Launcher launcher, String serverName, String token, JSONObject saltFunc, TaskListener listener) throws IOException, InterruptedException, SaltException { + if (saltFunc.get("client").equals("local_async")) { + return Builds.getBlockingBuildJid(launcher, serverName, token, saltFunc, listener); + } + return null; + } + public JSONArray performRequest(Launcher launcher, Run build, String token, String serverName, JSONObject saltFunc, TaskListener listener, String netapi) throws InterruptedException, IOException, SaltException { JSONArray returnArray = new JSONArray(); @@ -277,7 +286,8 @@ public JSONArray performRequest(Launcher launcher, Run build, String token, Stri int jobPollTime = getJobPollTime(); int minionTimeout = getMinionTimeout(); // poll /minion for response - returnArray = Builds.runBlockingBuild(launcher, build, serverName, token, saltFunc, listener, jobPollTime, minionTimeout, netapi); + String jid = getJID(launcher, serverName, token, saltFunc, listener); + returnArray = Builds.checkBlockingBuild(launcher, serverName, token, saltFunc, listener, jobPollTime, minionTimeout, netapi, jid); } else { // Just send a salt request to /. Don't wait for reply httpResponse = launcher.getChannel().call(new HttpCallable(serverName, saltFunc, token)); diff --git a/src/main/java/com/waytta/SaltAPIStep.java b/src/main/java/com/waytta/SaltAPIStep.java index bd260a6..f1d8e9a 100644 --- a/src/main/java/com/waytta/SaltAPIStep.java +++ b/src/main/java/com/waytta/SaltAPIStep.java @@ -2,18 +2,17 @@ import com.waytta.SaltException; -import java.io.IOException; -import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import java.io.IOException; +import java.io.Serializable; +import java.util.Set; +import java.util.concurrent.ScheduledFuture; import javax.inject.Inject; import hudson.model.Run; import hudson.model.TaskListener; -import org.jenkinsci.plugins.workflow.steps.AbstractStepImpl; -import org.jenkinsci.plugins.workflow.steps.AbstractStepDescriptorImpl; -import org.jenkinsci.plugins.workflow.steps.AbstractSynchronousStepExecution; -import org.jenkinsci.plugins.workflow.steps.StepContextParameter; +import org.jenkinsci.plugins.workflow.steps.*; import org.kohsuke.stapler.AncestorInPath; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; @@ -24,22 +23,18 @@ import hudson.Launcher; import hudson.model.Item; import hudson.model.Job; -import hudson.model.Result; -import hudson.tasks.BuildStepDescriptor; -import hudson.tasks.Builder; import hudson.util.FormValidation; import hudson.util.ListBoxModel; -import jenkins.model.Jenkins; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import com.cloudbees.plugins.credentials.CredentialsProvider; -import com.cloudbees.plugins.credentials.CredentialsMatchers; -import com.cloudbees.plugins.credentials.common.StandardListBoxModel; import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials; import com.waytta.clientinterface.BasicClient; -public class SaltAPIStep extends AbstractStepImpl { +import com.google.common.collect.ImmutableSet; + +public class SaltAPIStep extends Step implements Serializable { private static final Logger LOGGER = Logger.getLogger("com.waytta.saltstack"); private String servername; @@ -48,6 +43,9 @@ public class SaltAPIStep extends AbstractStepImpl { private boolean saveEnvVar = false; private final String credentialsId; private boolean saveFile = false; + private static String token = null; + private static String netapi = null; + private static JSONObject saltFunc = null; @DataBoundConstructor public SaltAPIStep(String servername, String authtype, BasicClient clientInterface, String credentialsId) { @@ -149,10 +147,7 @@ public DescriptorImpl getDescriptor() { } @Extension - public static final class DescriptorImpl extends AbstractStepDescriptorImpl { - public DescriptorImpl() { - super(SaltAPIStepExecution.class); - } + public static final class DescriptorImpl extends StepDescriptor { @Override public String getFunctionName() { @@ -186,27 +181,111 @@ public FormValidation doTestConnection( @AncestorInPath Item project) { return SaltAPIBuilder.DescriptorImpl.doTestConnection(servername, credentialsId, authtype, project); } + + @Override + public Set> getRequiredContext() { + return ImmutableSet.of(Run.class, FilePath.class, TaskListener.class, Launcher.class); + } } - public static class SaltAPIStepExecution extends AbstractSynchronousStepExecution { + @Override public StepExecution start(StepContext context) throws Exception { + return new Execution(this, context); + } + + public class Execution extends AbstractStepExecutionImpl { + private static final long serialVersionUID = 1L; + + private String jid; + @Inject - private transient SaltAPIStep saltStep; + private SaltAPIStep saltStep; - @StepContextParameter - private transient Run run; + private transient volatile ScheduledFuture task; - @StepContextParameter - private transient FilePath workspace; + SaltAPIBuilder saltBuilder; - @StepContextParameter - private transient TaskListener listener; + Execution(SaltAPIStep step, StepContext context) { + super(context); + this.saltStep = step; + } - @StepContextParameter - private transient Launcher launcher; + @Override + public void stop(Throwable cause) throws Exception { + if (task != null) { + task.cancel(false); + } + getContext().onFailure(cause); + } @Override - protected String run() throws Exception, SaltException { - SaltAPIBuilder saltBuilder = new SaltAPIBuilder(saltStep.servername, saltStep.authtype, saltStep.clientInterface, saltStep.credentialsId); + public boolean start() throws Exception { + Launcher launcher = getContext().get(Launcher.class); + TaskListener listener = getContext().get(TaskListener.class); + + prepareRun(); + jid = saltBuilder.getJID(launcher, saltBuilder.getServername(), token, saltFunc, listener); + + new Thread("saltAPI") { + @Override + public void run() { + try { + saltPerform(token, saltFunc, netapi); + } + catch (Exception e) { + Execution.this.getContext().onFailure(e); + } + } + }.start(); + + return false; + } + + @Override public void onResume() { + TaskListener listener = null; + Launcher launcher = null; + FilePath workspace = null; + try { + listener = getContext().get(TaskListener.class); + launcher = getContext().get(Launcher.class); + workspace = getContext().get(FilePath.class); + } catch (Exception e) { + Execution.this.getContext().onFailure(e); + } + + // Fail out if missing jid + if (jid == null || jid.equals("")) { + throw new RuntimeException("Unable to resume. Missing JID."); + } + + listener.getLogger().println("Resuming jid: " + jid); + + // Auth to salt-api + try { + prepareRun(); + } catch (Exception e) { + Execution.this.getContext().onFailure(e); + } + + // Poll for completion + int jobPollTime = saltBuilder.getJobPollTime(); + int minionTimeout = saltBuilder.getMinionTimeout(); + JSONArray returnArray = null; + try { + returnArray = Builds.checkBlockingBuild(launcher, saltBuilder.getServername(), token, saltFunc, listener, jobPollTime, minionTimeout, netapi, jid); + } catch (Exception e) { + Execution.this.getContext().onFailure(e); + } + + // Verify and return result + postRun(returnArray); + } + + private void prepareRun() throws InterruptedException, IOException{ + Runrun = getContext().get(Run.class); + TaskListener listener = getContext().get(TaskListener.class); + Launcher launcher = getContext().get(Launcher.class); + + saltBuilder = new SaltAPIBuilder(saltStep.servername, saltStep.authtype, saltStep.clientInterface, saltStep.credentialsId); StandardUsernamePasswordCredentials credential = CredentialsProvider.findCredentialById( saltBuilder.getCredentialsId(), StandardUsernamePasswordCredentials.class, run); @@ -219,32 +298,60 @@ protected String run() throws Exception, SaltException { // Get an auth token ServerToken serverToken = Utils.getToken(launcher, saltBuilder.getServername(), auth); - String token = serverToken.getToken(); - String netapi = serverToken.getServer(); + token = serverToken.getToken(); + netapi = serverToken.getServer(); LOGGER.log(Level.FINE, "Discovered netapi: " + netapi); // If we got this far, auth must have been good and we've got a token - JSONObject saltFunc = saltBuilder.prepareSaltFunction(run, listener, saltBuilder.getClientInterface().getDescriptor().getDisplayName(), saltBuilder.getTarget(), saltBuilder.getFunction(), saltBuilder.getArguments()); + saltFunc = saltBuilder.prepareSaltFunction(run, listener, saltBuilder.getClientInterface().getDescriptor().getDisplayName(), saltBuilder.getTarget(), saltBuilder.getFunction(), saltBuilder.getArguments()); LOGGER.log(Level.FINE, "Sending JSON: " + saltFunc.toString()); + } + + private void saltPerform(String token, JSONObject saltFunc, String netapi) throws Exception, SaltException { + Runrun = getContext().get(Run.class); + FilePath workspace = getContext().get(FilePath.class); + TaskListener listener = getContext().get(TaskListener.class); + Launcher launcher = getContext().get(Launcher.class); + + JSONArray returnArray = null; + if (jid != null && !jid.equals("")) { + returnArray = Builds.checkBlockingBuild(launcher, saltBuilder.getServername(), token, saltFunc, listener, saltBuilder.getJobPollTime(), saltBuilder.getMinionTimeout(), netapi, jid); + } else { + returnArray = saltBuilder.performRequest(launcher, run, token, saltBuilder.getServername(), saltFunc, listener, netapi); + } + postRun(returnArray); + } + + private void postRun(JSONArray returnArray) { + TaskListener listener = null; + FilePath workspace = null; + try { + listener = getContext().get(TaskListener.class); + workspace = getContext().get(FilePath.class); + } catch (Exception e) { + Execution.this.getContext().onFailure(e); + } - JSONArray returnArray = saltBuilder.performRequest(launcher, run, token, saltBuilder.getServername(), saltFunc, listener, netapi); LOGGER.log(Level.FINE, "Received response: " + returnArray); // Check for error and print out results boolean validFunctionExecution = Utils.validateFunctionCall(returnArray); if (!validFunctionExecution) { listener.error("One or more minion did not return code 0\n"); - throw new SaltException(returnArray.toString()); + Execution.this.getContext().onFailure(new SaltException(returnArray.toString())); } if (saltStep.saveFile) { - Utils.writeFile(returnArray.toString(), workspace); + try { + Utils.writeFile(returnArray.toString(), workspace); + } catch (Exception e) { + Execution.this.getContext().onFailure(e); + } } - return returnArray.toString(); + // Return results + getContext().onSuccess(returnArray.toString()); } - - private static final long serialVersionUID = 1L; } } \ No newline at end of file diff --git a/src/main/java/com/waytta/Utils.java b/src/main/java/com/waytta/Utils.java index 383b260..a11409e 100644 --- a/src/main/java/com/waytta/Utils.java +++ b/src/main/java/com/waytta/Utils.java @@ -11,8 +11,6 @@ import hudson.Launcher; import hudson.model.Run; import hudson.model.TaskListener; -import jenkins.model.Jenkins; - import net.sf.json.JSONArray; import net.sf.json.JSONException; import net.sf.json.JSONObject; diff --git a/src/main/java/com/waytta/clientinterface/BasicClient.java b/src/main/java/com/waytta/clientinterface/BasicClient.java index 8fdf5e0..b6e5e1e 100644 --- a/src/main/java/com/waytta/clientinterface/BasicClient.java +++ b/src/main/java/com/waytta/clientinterface/BasicClient.java @@ -4,72 +4,73 @@ import hudson.model.Describable; import hudson.model.Descriptor; import jenkins.model.Jenkins; -import org.kohsuke.stapler.DataBoundConstructor; -import hudson.DescriptorExtensionList; +import java.io.Serializable; import java.util.List; import hudson.util.ListBoxModel; -abstract public class BasicClient implements ExtensionPoint, Describable { - public String getFunction() { - return ""; - } +abstract public class BasicClient implements ExtensionPoint, Serializable, Describable { + private static final long serialVersionUID = 1L; - public String getArguments() { - return ""; - } + public String getFunction() { + return ""; + } - public String getTarget() { - return ""; - } + public String getArguments() { + return ""; + } - public String getTargettype() { - return ""; - } + public String getTarget() { + return ""; + } - public boolean getBlockbuild() { - return false; - } + public String getTargettype() { + return ""; + } - public String getBatchSize() { - return ""; - } + public boolean getBlockbuild() { + return false; + } - public int getJobPollTime() { - return 10; - } + public String getBatchSize() { + return ""; + } + + public int getJobPollTime() { + return 10; + } - public int getMinionTimeout() { - return 30; - } + public int getMinionTimeout() { + return 30; + } - public String getMods() { - return ""; - } + public String getMods() { + return ""; + } - public String getPillarvalue() { - return ""; - } + public String getPillarvalue() { + return ""; + } - public String getSubset() { - return "1"; - } + public String getSubset() { + return "1"; + } - public String getTag() { - return ""; - } + public String getTag() { + return ""; + } - public String getPost() { - return ""; - } + public String getPost() { + return ""; + } - @Override + @Override public Descriptor getDescriptor() { - Jenkins jenkins = Jenkins.getInstance(); - if (jenkins == null) { - throw new IllegalStateException("Jenkins has not been started, or was already shut down"); - } - return jenkins.getDescriptorOrDie(getClass()); + Jenkins jenkins = Jenkins.getInstance(); + if (jenkins == null) { + throw new IllegalStateException("Jenkins has not been started, or was already shut down"); + } + return jenkins.getDescriptorOrDie(getClass()); } public List getClientDescriptors() {