Merge branch 'master' into zkofiro/SRE-30670

agent/agent_common/pom.xml

@@ -29,7 +29,7 @@
 
     <dependency>
       <groupId>org.jdom</groupId>
-      <artifactId>jdom</artifactId>
+      <artifactId>jdom2</artifactId>
     </dependency>
 
     <dependency>

agent/agent_standalone/src/main/java/com/intuit/tank/standalone/agent/StandaloneAgentStartup.java

@@ -18,6 +18,7 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -97,9 +98,8 @@ public void startTest(final StandaloneAgentRequest request) {
                     }
                 }
                 // now start the harness
-                String cmd = API_HARNESS_COMMAND + " -http=" + controllerBase + " -jobId=" + request.getJobId()
-                        + " -stopBehavior=" + request.getStopBehavior();
-                LOG.info("Starting apiharness with command: " + cmd);
+                String[] cmd = {API_HARNESS_COMMAND, " -http=", controllerBase, " -jobId=", request.getJobId(), " -stopBehavior=", request.getStopBehavior()};
+                LOG.info("Starting apiharness with command: {}", Arrays.toString(cmd));
                 currentAvailability.setAvailabilityStatus(AgentAvailabilityStatus.RUNNING_JOB);
                 sendAvailability();
                 Process exec = Runtime.getRuntime().exec(cmd);
@@ -108,7 +108,7 @@ public void startTest(final StandaloneAgentRequest request) {
                 sendAvailability();
                 //
             } catch (Exception e) {
-                LOG.error("Error in AgentStartup " + e, e);
+                LOG.error("Error in AgentStartup {}", e, e);
                 currentAvailability.setAvailabilityStatus(AgentAvailabilityStatus.AVAILABLE);
                 try {
                     sendAvailability();

agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java

@@ -94,7 +94,8 @@ public void run() {
             String jvmArgs = AmazonUtil.getUserDataAsMap().get(TankConstants.KEY_JVM_ARGS);
             logger.info("Starting apiharness with command: "
                     + API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs);
-            Runtime.getRuntime().exec(API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs);
+            Runtime.getRuntime().exec(
+                    new String[] {API_HARNESS_COMMAND, "-http=", controllerBaseUrl, jvmArgs});
         } catch (ConnectException ce) {
             logger.error("Error creating connection to "
                     + controllerBaseUrl + " : this is normal during the bake : " + ce.getMessage());

agent/apiharness/src/main/java/com/intuit/tank/harness/APIMonitor.java

@@ -74,9 +74,12 @@ public void run() {
                 }
                 if (!isLocal) setInstanceStatus(newStatus.getInstanceId(), newStatus);
                 APITestHarness.getInstance().checkAgentThreads();
-                Thread.sleep(reportInterval);
             } catch (Exception t) {
                 LOG.error(LogUtil.getLogMessage("Unable to send status metrics | " + t.getMessage()), t);
+            } finally {
+                try {
+                    Thread.sleep(reportInterval);
+                } catch ( InterruptedException ie) { /*Ignore*/ }
             }
         }
     }

api/src/main/java/com/intuit/tank/jobs/models/CreateJobRegion.java

@@ -7,6 +7,7 @@
  */
 package com.intuit.tank.jobs.models;
 
+import com.intuit.tank.projects.models.Namespace;
 import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlRootElement;
 
@@ -18,12 +19,16 @@ public class CreateJobRegion {
 
 	@XmlElement(name="users")
 	private String users;
+
+	@XmlElement(name = "percentage")
+	private String percentage;
 
 	public CreateJobRegion() {}
 
-	public CreateJobRegion(String region, String users) {
+	public CreateJobRegion(String region, String users, String percentage) {
 		this.region = region;
 		this.users = users;
+		this.percentage = percentage;
 	}
 
     /**
@@ -39,4 +44,11 @@ public String getRegion() {
     public String getUsers() {
         return users;
     }
+
+	/**
+	 * @return the percentage
+	 */
+	public String getPercentage() {
+		return percentage;
+	}
 }

api/src/main/java/com/intuit/tank/jobs/models/CreateJobRequest.java

@@ -8,6 +8,8 @@
 package com.intuit.tank.jobs.models;
 
 import com.intuit.tank.harness.StopBehavior;
+import com.intuit.tank.projects.models.Namespace;
+import com.intuit.tank.vm.api.enumerated.IncrementStrategy;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 
@@ -39,7 +41,10 @@ public class CreateJobRequest implements Serializable {
 
     @XmlElement(name="userIntervalIncrement")
     private int userIntervalIncrement;
-
+
+    @XmlElement(name="targetRampRate")
+    private double targetRampRate;
+
     @XmlElement(name="stopBehavior")
     private String stopBehavior = StopBehavior.END_OF_TEST.name();
 
@@ -48,10 +53,16 @@ public class CreateJobRequest implements Serializable {
 
     @XmlElement(name="numUsersPerAgent")
     private int numUsersPerAgent;
+
+    @XmlElement(name="targetRatePerAgent")
+    private double targetRatePerAgent;
 
     @XmlElement(name="jobRegions")
     private Set<CreateJobRegion> jobRegions = new HashSet<CreateJobRegion>();
 
+    @XmlElement(name = "workloadType")
+    private IncrementStrategy workloadType;
+
     public CreateJobRequest(String projectName) {
     	this.projectName = projectName;
     }
@@ -100,14 +111,27 @@ public int getUserIntervalIncrement() {
         return userIntervalIncrement;
     }
 
+    /**
+     * @return the targetRampRate
+     */
+    public double getTargetRampRate() {
+        return targetRampRate;
+    }
+
     /**
      * @return the jobRegions
      */
     public Set<CreateJobRegion> getJobRegions() {
         return jobRegions;
     }
-
-
+
+    /**
+     * @return the workloadType (linear / nonlinear)
+     */
+    public IncrementStrategy getWorkloadType() {
+        return workloadType;
+    }
+
     /**
      * @return the stopBehavior
      */
@@ -128,7 +152,12 @@ public String getVmInstance() {
     public int getNumUsersPerAgent() {
         return numUsersPerAgent;
     }
-
+
+    /**
+     * @return the targetRatePerAgent
+     */
+    public double getTargetRatePerAgent() { return targetRatePerAgent; }
+
     /**
      * @param jobRegions
      *            the jobRegions to set

data_model/src/main/java/com/intuit/tank/project/BaseJob.java

@@ -71,7 +71,7 @@ public abstract class BaseJob extends BaseEntity {
     @Column(name = "num_agents", columnDefinition = "INT(11) NOT NULL DEFAULT '1'")
     private int numAgents = 1;
 
-    @Column(name = "target_rate_per_agent", columnDefinition="DECIMAL(10,2) NOT NULL DEFAULT '1.00'")
+    @Column(name = "target_rate_per_agent", columnDefinition="DECIMAL(10,4) NOT NULL DEFAULT '1.00'")
     private Double targetRatePerAgent = 1.00;
 
     @Column(name = "logging_profile")

pom.xml

@@ -33,7 +33,7 @@
     <version.software.amazon.awssdk>2.25.56</version.software.amazon.awssdk>
     <version.aws-xray-sdk-bom>2.15.3</version.aws-xray-sdk-bom>
 
-    <version.tomcat>10.1.24</version.tomcat>
+    <version.tomcat>10.1.25</version.tomcat>
     <weld.version>5.1.2.Final</weld.version>
     <cdi.version>4.1.0</cdi.version>
 
@@ -632,11 +632,6 @@
         <artifactId>serializer</artifactId>
         <version>2.7.3</version>
       </dependency>
-      <dependency>
-        <groupId>axis</groupId>
-        <artifactId>axis</artifactId>
-        <version>1.4</version>
-      </dependency>
       <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-core</artifactId>
@@ -660,8 +655,8 @@
       </dependency>
       <dependency>
         <groupId>org.jdom</groupId>
-        <artifactId>jdom</artifactId>
-        <version>2.0.2</version>
+        <artifactId>jdom2</artifactId>
+        <version>2.0.6.1</version>
       </dependency>
       <dependency>
         <groupId>jakarta.xml</groupId>

rest-mvc/impl/src/main/java/com/intuit/tank/rest/mvc/rest/controllers/JobController.java

@@ -90,9 +90,11 @@ public ResponseEntity<JobContainer> getJobsByProject(@PathVariable @Parameter(de
             "  - rampTime and simulationTime are accepted as time strings i.e 60s, 12m, 24h \n\n" +
             "  - stopBehavior is matched against accepted values ( END_OF_STEP,  END_OF_SCRIPT,  END_OF_SCRIPT_GROUP,  END_OF_TEST ) \n\n" +
             "  - vmInstance matches against AWS EC2 Instance Types i.e c5.large, c5.xlarge, etc \n\n"+
+            "  - workloadType can be set to increasing (linear workload) or standard (nonlinear workload)  \n\n"+
+            "  - targetRampRate, targetRatePerAgent, and jobRegions.percentage fields apply to standard workloadType jobs (nonlinear) \n\n"+
             "  - projectId, userIntervalIncrement and numUsersPerAgent are accepted as integers \n\n" +
             "  - jobRegions.regions correspond to AWS regions in lowercase i.e us-west-2, us-east-2 \n\n" +
-            "  - jobRegions.users are accepted as integer strings i.e \"100\", \"4000\" \n\n", summary =  "Create a new job")
+            "  - jobRegions.users and jobRegions.percentage are accepted as integer strings i.e \"100\", \"4000\" \n\n", summary =  "Create a new job")
     @ApiResponses(value = {
             @ApiResponse(responseCode = "201", description = "Successfully created job", content = @Content),
             @ApiResponse(responseCode = "400", description = "Could not create job due to bad request", content = @Content)

rest-mvc/impl/src/main/java/com/intuit/tank/rest/mvc/rest/services/jobs/JobServiceV2Impl.java

@@ -30,6 +30,7 @@
 import com.intuit.tank.rest.mvc.rest.controllers.errors.GenericServiceCreateOrUpdateException;
 import com.intuit.tank.rest.mvc.rest.controllers.errors.GenericServiceInternalServerException;
 import com.intuit.tank.rest.mvc.rest.controllers.errors.GenericServiceResourceNotFoundException;
+import com.intuit.tank.vm.api.enumerated.IncrementStrategy;
 import com.intuit.tank.vm.vmManager.models.CloudVmStatusContainer;
 import com.intuit.tank.jobs.models.JobContainer;
 import com.intuit.tank.jobs.models.JobTO;
@@ -311,7 +312,13 @@ public String killJob(Integer jobId) {
     // Job Service Util
 
     public static void buildJobConfiguration(@Nonnull CreateJobRequest request, Project project) {
-        JobConfiguration jobConfiguration = project.getWorkloads().get(0).getJobConfiguration();
+        JobConfiguration jobConfiguration = (project != null && project.getWorkloads() != null && !project.getWorkloads().isEmpty())
+                ? project.getWorkloads().get(0).getJobConfiguration()
+                : null;
+
+        if(jobConfiguration == null){
+           throw new GenericServiceCreateOrUpdateException("jobs", "job", null);
+        }
 
         if (StringUtils.isNotEmpty(request.getRampTime())) {
             jobConfiguration.setRampTimeExpression(request.getRampTime());
@@ -321,11 +328,20 @@ public static void buildJobConfiguration(@Nonnull CreateJobRequest request, Proj
             jobConfiguration.setSimulationTimeExpression(request.getSimulationTime());
         }
 
+        if(request.getWorkloadType() != null) {
+            jobConfiguration.setIncrementStrategy(request.getWorkloadType());
+        }
+
         jobConfiguration.setUserIntervalIncrement(request.getUserIntervalIncrement());
+        jobConfiguration.setTargetRampRate(request.getTargetRampRate());
+        jobConfiguration.setTargetRatePerAgent(request.getTargetRatePerAgent());
+
         jobConfiguration.setStopBehavior(StringUtils.isNotEmpty(request.getStopBehavior())
                 ? request.getStopBehavior() : StopBehavior.END_OF_SCRIPT_GROUP.name());
 
-        jobConfiguration.setVmInstanceType(request.getVmInstance());
+        if (StringUtils.isNotEmpty(request.getVmInstance())) {
+            jobConfiguration.setVmInstanceType(request.getVmInstance());
+        }
         jobConfiguration.setNumUsersPerAgent(request.getNumUsersPerAgent());
 
         boolean hasSimTime = jobConfiguration.getSimulationTime() > 0
@@ -340,13 +356,20 @@ public static void buildJobConfiguration(@Nonnull CreateJobRequest request, Proj
     }
 
     private static void setJobRegions(@Nonnull CreateJobRequest request, JobConfiguration jobConfiguration) {
+        if(jobConfiguration == null || jobConfiguration.getJobRegions() == null){
+            return;
+        }
         jobConfiguration.getJobRegions().clear();
         JobRegionDao jrd = new JobRegionDao();
-        for (CreateJobRegion r : request.getJobRegions()) {
-            if (StringUtils.isNotEmpty(r.getRegion())) {
-                JobRegion jr = jrd.saveOrUpdate(
-                        new JobRegion(VMRegion.getRegionFromZone(r.getRegion()), r.getUsers()));
-                jobConfiguration.getJobRegions().add(jr);
+        if (request.getJobRegions() != null) {
+            for (CreateJobRegion r : request.getJobRegions()) {
+                if (StringUtils.isNotEmpty(r.getRegion())) {
+                    String users = r.getUsers() != null ? r.getUsers() : "0";
+                    String percentage = r.getPercentage() != null ? r.getPercentage() : "0";
+                    JobRegion jr = jrd.saveOrUpdate(
+                            new JobRegion(VMRegion.getRegionFromZone(r.getRegion()), users, percentage));
+                    jobConfiguration.getJobRegions().add(jr);
+                }
             }
         }
     }
@@ -406,9 +429,12 @@ public static JobInstance addJobToQueue(Project project, CreateJobRequest reques
 
     private static String buildJobInstanceName(CreateJobRequest request, Workload workload, Project project) {
         String projectName = request.getProjectName() == null ? project.getName() : request.getProjectName();
+        String jobType = "_nonlinear_";
+        if(request.getWorkloadType().equals(IncrementStrategy.increasing)) {
+            jobType = "_" + workload.getJobConfiguration().getTotalVirtualUsers() + "_users_"; // set to total users for linear
+        }
         return StringUtils.isNotEmpty(request.getJobInstanceName()) ? request.getJobInstanceName()
-                : projectName + "_" + workload.getJobConfiguration().getTotalVirtualUsers() + "_users_"
-                + ReportUtil.getTimestamp(new Date());
+                : projectName + jobType + ReportUtil.getTimestamp(new Date());
     }
 
     private static Set<EntityVersion> getVersions(BaseDao dao, Collection<Integer> dataFileIds,

script_processor/pom.xml

@@ -58,7 +58,7 @@
 
     <dependency>
       <groupId>org.jdom</groupId>
-      <artifactId>jdom</artifactId>
+      <artifactId>jdom2</artifactId>
       <scope>test</scope>
     </dependency>
 

tank_vmManager/src/main/java/com/intuit/tank/perfManager/workLoads/JobManager.java

@@ -272,10 +272,11 @@ protected List<String> getInstanceUrl(List<String> instanceIds) {
      * @return AgentData
      */
     private AgentData findAgent(String instanceId) {
-        for (VMRegion region : tankConfig.getVmManagerConfig().getConfiguredRegions()) {
-            String instanceUrl = new AmazonInstance(region).findDNSName(instanceId);
-            if (StringUtils.isNotEmpty(instanceUrl)) {
-                instanceUrl = "http://" + instanceUrl + ":" + tankConfig.getAgentConfig().getAgentPort();
+        String instanceUrl;
+        for (VMRegion region : tankConfig.getVmManagerConfig().getRegions()) {
+            Optional<String> instanceUrlOptional = new AmazonInstance(region).findDNSName(instanceId);
+            if (instanceUrlOptional.isPresent()) {
+                instanceUrl = "http://" + instanceUrlOptional.get() + ":" + tankConfig.getAgentConfig().getAgentPort();
                 return new AgentData("0", instanceId, instanceUrl, 0, region, "zone");
             }
         }

tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/amazon/AmazonInstance.java

@@ -43,16 +43,7 @@
 
 import jakarta.annotation.Nonnull;
 import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.stream.Collectors;
@@ -559,20 +550,22 @@ private AssociateContainer(String instanceId, Address address) {
 
     }
 
-    public String findDNSName(String instanceId) {
+    public Optional<String> findDNSName(String instanceId) {
         try {
-            return ec2client.describeInstances().reservations().stream()
+            Optional<String> dnsName =  ec2client.describeInstances().reservations().stream()
                     .flatMap(reservationDescription -> reservationDescription.instances().stream())
                     .filter(instance -> instanceId.equals(instance.instanceId()))
                     .findFirst()
                     .map(instance -> (StringUtils.isNotEmpty(instance.publicDnsName()))
                             ? instance.publicDnsName()
-                            : instance.privateDnsName())
-                    .toString();
+                            : instance.privateDnsName());
+            if (dnsName.isPresent()) {
+                return dnsName;
+            }
         } catch (Exception e) {
             LOG.error("Error getting public dns in " + vmRegion + ": " + e.getMessage());
         }
-        return null;
+        return Optional.empty();
     }
 
 }