Proposal: add tooling for memory load tests

arquillian.xml

@@ -2,7 +2,7 @@
 <arquillian xmlns="http://jboss.org/schema/arquillian" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://jboss.org/schema/arquillian http://jboss.org/schema/arquillian/arquillian_1_0.xsd">
 
-    <container qualifier="jboss" default="true" >
+    <container qualifier="jboss" default="true" mode="manual">
         <configuration>
             <property name="jbossHome">${basedir}/target/jboss-as</property>
             <property name="javaVmArguments">-server -Xms64m -Xmx512m</property>

microprofile-health/pom.xml

@@ -44,5 +44,11 @@
             <artifactId>tooling-server-configuration</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.jboss.eap.qe</groupId>
+            <artifactId>tooling-performance</artifactId>
+            <version>1.0.0.Final-SNAPSHOT</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

microprofile-health/src/test/java/org/jboss/eap/qe/microprofile/health/performance/MultipleSubsequentDeploymentMemoryStressTest.java

@@ -0,0 +1,178 @@
+package org.jboss.eap.qe.microprofile.health.performance;
+
+import static io.restassured.RestAssured.get;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.not;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.container.test.api.Deployer;
+import org.jboss.arquillian.container.test.api.Deployment;
+import org.jboss.arquillian.container.test.api.RunAsClient;
+import org.jboss.arquillian.junit.Arquillian;
+import org.jboss.arquillian.junit.InSequence;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.eap.qe.microprofile.health.BothHealthCheck;
+import org.jboss.eap.qe.microprofile.health.LivenessHealthCheck;
+import org.jboss.eap.qe.microprofile.health.ReadinessHealthCheck;
+import org.jboss.eap.qe.microprofile.health.performance.evaluation.IncreaseOverToleranceEvaluator;
+import org.jboss.eap.qe.microprofile.tooling.performance.core.Gauge;
+import org.jboss.eap.qe.microprofile.tooling.performance.core.MeasurementException;
+import org.jboss.eap.qe.microprofile.tooling.performance.core.StressTestException;
+import org.jboss.eap.qe.microprofile.tooling.performance.core.StressTestProtocol;
+import org.jboss.eap.qe.microprofile.tooling.performance.core.StressTester;
+import org.jboss.eap.qe.microprofile.tooling.performance.memory.JMXBasedMemoryGauge;
+import org.jboss.eap.qe.microprofile.tooling.performance.memory.MemoryUsageRecord;
+import org.jboss.eap.qe.microprofile.tooling.performance.protocol.MultipleRepeatableActionsProtocol;
+import org.jboss.eap.qe.microprofile.tooling.server.configuration.ConfigurationException;
+import org.jboss.eap.qe.microprofile.tooling.server.configuration.arquillian.ArquillianContainerProperties;
+import org.jboss.eap.qe.microprofile.tooling.server.configuration.arquillian.ArquillianDescriptorWrapper;
+import org.jboss.eap.qe.microprofile.tooling.server.configuration.creaper.ManagementClientProvider;
+import org.jboss.shrinkwrap.api.Archive;
+import org.jboss.shrinkwrap.api.ShrinkWrap;
+import org.jboss.shrinkwrap.api.asset.EmptyAsset;
+import org.jboss.shrinkwrap.api.spec.WebArchive;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.wildfly.extras.creaper.core.online.OnlineManagementClient;
+
+/**
+ * Memory stress test cases for multiple subsequent re-deployments
+ */
+@RunWith(Arquillian.class)
+@RunAsClient
+public class MultipleSubsequentDeploymentMemoryStressTest {
+
+    private final static String DEPLOYMENT_NAME = "deployment-1";
+    private final static int POST_DEPLOYMENT_GRACEFUL_WAIT_TIME_IN_MSEC = 1000;
+    private final static int INCREASED_MEMORY_FOOTPRINT_TOLERANCE_PERCENT = 7;
+    private final static int SIMPLE_REDEPLOY_ITERATIONS = 100;
+    private final static int PROBING_INTERVAL_SIMPLE_REDEPLOY_ITERATIONS = SIMPLE_REDEPLOY_ITERATIONS;
+    private final static int LINEAR_FIT_SLOPE_COMPARISON_REDEPLOY_ITERATIONS = 64;
+    private final static int PROBING_INTERVAL_LINEAR_FIT_SLOPE_COMPARISON_REDEPLOY_ITERATIONS = 8;
+    private final static long MEGABYTE = 1024 * 1024;
+    private final static long MEMORY_INCREASE_TOLERANCE_IN_MB = 25 * MEGABYTE;
+
+    private static ArquillianContainerProperties arquillianContainerProperties = new ArquillianContainerProperties(
+            ArquillianDescriptorWrapper.getArquillianDescriptor());
+    private static OnlineManagementClient onlineManagementClient;
+    private static String healthUrl;
+
+    private static Gauge<MemoryUsageRecord> gauge;
+
+    @ArquillianResource
+    private Deployer deployer;
+
+    @ArquillianResource
+    private ContainerController contoller;
+
+    @BeforeClass
+    public static void setup()
+            throws ConfigurationException, IOException {
+
+        onlineManagementClient = ManagementClientProvider.onlineStandalone();
+
+        healthUrl = String.format("http://%s:%d/health",
+                arquillianContainerProperties.getDefaultManagementAddress(),
+                arquillianContainerProperties.getDefaultManagementPort());
+
+        gauge = new JMXBasedMemoryGauge(arquillianContainerProperties.getDefaultManagementAddress(),
+                arquillianContainerProperties.getDefaultManagementPort());
+    }
+
+    @AfterClass
+    public static void tearDown() throws IOException {
+        onlineManagementClient.close();
+    }
+
+    @Deployment(name = DEPLOYMENT_NAME, managed = false, testable = false)
+    public static Archive<?> serviceProviderDeployment() {
+        WebArchive deployment = ShrinkWrap.create(
+                WebArchive.class, DEPLOYMENT_NAME + ".war")
+                .addClasses(BothHealthCheck.class, LivenessHealthCheck.class, ReadinessHealthCheck.class)
+                .addAsWebInfResource(EmptyAsset.INSTANCE, "beans.xml");
+        return deployment;
+    }
+
+    private Void executeSimpleRedeployActions() throws InterruptedException {
+
+        deployer.deploy(DEPLOYMENT_NAME);
+        Thread.sleep(POST_DEPLOYMENT_GRACEFUL_WAIT_TIME_IN_MSEC);
+        String response = get(healthUrl)
+                .then()
+                .statusCode(200)
+                .body(not(empty()))
+                .extract().asString();
+        System.out.println(response);
+        deployer.undeploy(DEPLOYMENT_NAME);
+
+        return null;
+    }
+
+    /**
+     * @tpTestDetails Test to verify that a number of subsequent deployments doesn't cause memory leaks.
+     *                This stress test executes the multiple subsequent re-deployments protocol and measures memory
+     *                footprint at the beginning and at the end.
+     * @tpPassCrit Final value does not exceed initial value by more than
+     *             {@link MultipleSubsequentDeploymentMemoryStressTest#MEMORY_INCREASE_TOLERANCE_IN_MB}
+     * @tpSince EAP 7.4.0.CD19
+     */
+    @Test
+    @InSequence(1)
+    public void testSimpleSeveralRedeployProtocol() throws StressTestException {
+
+        //  we have one tester which is going to take care of this stress test, it registers measurements
+        //  as instances of MemoryUsageRecord and uses a gauge that accepts this data type
+        StressTester<MemoryUsageRecord, Gauge<MemoryUsageRecord>> tester = new StressTester(gauge);
+
+        //  this evaluator is intended to assess whether final value is showing an increase bigger than the
+        //  accepted tolerance when compared to initial value
+        IncreaseOverToleranceEvaluator evaluator = new IncreaseOverToleranceEvaluator(MEMORY_INCREASE_TOLERANCE_IN_MB);
+
+        //  let's start with the test session: it defines a MultipleSubsequentDeploymentsProtocol to execute
+        //  SIMPLE_REDEPLOY_ITERATIONS redeploy actions and will probe for memory footprint each
+        //  PROBING_INTERVAL_SIMPLE_REDEPLOY_ITERATIONS attempts
+        StressTestProtocol simpleRedeployProtocol = new MultipleRepeatableActionsProtocol(
+                SIMPLE_REDEPLOY_ITERATIONS,
+                PROBING_INTERVAL_SIMPLE_REDEPLOY_ITERATIONS,
+                new Callable<Void>() {
+                    @Override
+                    public Void call() throws Exception {
+                        return executeSimpleRedeployActions();
+                    }
+                });
+        //  start the container
+        contoller.start("jboss");
+
+        //  initial value
+        try {
+            tester.probe();
+        } catch (MeasurementException e) {
+            throw new StressTestException(e);
+        }
+
+        //  let the tester execute its test session following the protocol
+        tester.executeSession(simpleRedeployProtocol);
+        //  stop the container
+        contoller.stop("jboss");
+
+        //   report control values to the evaluator
+        evaluator.setInitialValue(tester.getCollectedValues().get(0).getHeapSpaceUsed());
+        evaluator.setFinalValue(tester.getCollectedValues().get(tester.getCollectedValues().size() - 1).getHeapSpaceUsed());
+
+        //  let's evaluate results for this initial stress test session using IncreaseOverToleranceEvaluator
+        IncreaseOverToleranceEvaluator.Outcome outcome = evaluator.evaluate();
+        Long initialValue = outcome.getInitialValue(), finalValue = outcome.getFinalValue();
+        Assert.assertTrue(
+                String.format(
+                        "Memory consumption increase exceeds tolerance: (%s - %s) = %s > %s",
+                        initialValue, finalValue, finalValue - initialValue, MEMORY_INCREASE_TOLERANCE_IN_MB,
+                        INCREASED_MEMORY_FOOTPRINT_TOLERANCE_PERCENT),
+                outcome.isPassed());
+    }
+}

microprofile-health/src/test/java/org/jboss/eap/qe/microprofile/health/performance/evaluation/IncreaseOverToleranceEvaluator.java

@@ -0,0 +1,80 @@
+package org.jboss.eap.qe.microprofile.health.performance.evaluation;
+
+import org.jboss.eap.qe.microprofile.tooling.performance.core.StressTestEvaluator;
+import org.jboss.eap.qe.microprofile.tooling.performance.core.StressTestOutcome;
+
+/**
+ * This evaluator is intended to assess whether final value is showing an increase bigger than the
+ * accepted tolerance when compared to initial value
+ */
+public class IncreaseOverToleranceEvaluator implements StressTestEvaluator<IncreaseOverToleranceEvaluator.Outcome> {
+
+    private final Long tolerance;
+    private Long initialValue;
+    private Long finalValue;
+
+    public IncreaseOverToleranceEvaluator(final Long tolerance) {
+        this.tolerance = tolerance;
+    }
+
+    public Long getInitialValue() {
+        return initialValue;
+    }
+
+    public void setInitialValue(Long initialValue) {
+        this.initialValue = initialValue;
+    }
+
+    public Long getFinalValue() {
+        return finalValue;
+    }
+
+    public void setFinalValue(Long finalValue) {
+        this.finalValue = finalValue;
+    }
+
+    /**
+     * Outcome for this evaluator execution
+     */
+    public static class Outcome implements StressTestOutcome {
+
+        private final Long initialValue;
+        private final Long tolerance;
+        private final Long finalValue;
+        private final boolean passed;
+
+        Outcome(final Long initialValue, final Long finalValue, final Long tolerance, final boolean passed) {
+            this.initialValue = initialValue;
+            this.tolerance = tolerance;
+            this.finalValue = finalValue;
+            this.passed = passed;
+        }
+
+        public Long getInitialValue() {
+            return initialValue;
+        }
+
+        public Long getFinalValue() {
+            return finalValue;
+        }
+
+        public boolean isPassed() {
+            return passed;
+        }
+
+        static Outcome success(final Long initialValue, final Long finalValue, final Long tolerance) {
+            return new Outcome(initialValue, finalValue, tolerance, true);
+        }
+
+        static Outcome fail(final Long initialValue, final Long finalValue, final Long tolerance) {
+            return new Outcome(initialValue, finalValue, tolerance, false);
+        }
+    }
+
+    public Outcome evaluate() {
+        if ((finalValue - initialValue) > tolerance) {
+            return Outcome.fail(initialValue, finalValue, tolerance);
+        }
+        return Outcome.success(initialValue, finalValue, tolerance);
+    }
+}

microprofile-open-api/pom.xml

@@ -56,6 +56,12 @@
             <artifactId>tooling-server-configuration</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.jboss.eap.qe</groupId>
+            <artifactId>tooling-performance</artifactId>
+            <version>1.0.0.Final-SNAPSHOT</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>