[Backport 2.x] Integration tests framework (opensearch-project#3388)

Test framework that allows for declaring within the test class all
related users, roles, and other security primitives.  Emphasis on
hamcrest Matchers for straight forward to read, author, and
troubleshoot test cases.

Signed-off-by: Peter Nied <[email protected]>
Signed-off-by: Peter Nied <[email protected]>

.github/workflows/ci.yml

@@ -1,6 +1,12 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+      - 1.*
+      - 2.*
+  pull_request:
 
 env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
@@ -18,7 +24,7 @@ jobs:
         java-version: 17
 
     - name: Checkout security
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Generate list of tasks
       id: set-matrix
@@ -44,15 +50,14 @@ jobs:
         java-version: ${{ matrix.jdk }}
 
     - name: Checkout security
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Build and Test
       uses: gradle/gradle-build-action@v2
       with:
         cache-disabled: true
         arguments: |
           ${{ matrix.gradle_task }} -Dbuild.snapshot=false
-          -x test
 
     - name: Coverage
       uses: Wandalen/[email protected]
@@ -76,6 +81,32 @@ jobs:
       if: always()
       run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
 
+  integration-tests:
+    name: integration-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: [ubuntu-latest] # Removing windows temporarily
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v3
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout security
+      uses: actions/checkout@v4
+
+    - name: Build and Test
+      uses: gradle/gradle-build-action@v2
+      with:
+        cache-disabled: true
+        arguments: |
+          integrationTest -Dbuild.snapshot=false
+
   backward-compatibility:
     strategy:
       fail-fast: false
@@ -91,7 +122,7 @@ jobs:
         java-version: ${{ matrix.jdk }}
 
     - name: Checkout Security Repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - id: build-previous
       uses: ./.github/actions/run-bwc-suite
@@ -105,21 +136,21 @@ jobs:
   code-ql:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-java@v3
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: 11
-    - uses: github/codeql-action/init@v1
+    - uses: github/codeql-action/init@v2
       with:
         languages: java
-    - run: ./gradlew clean build -Dbuild.snapshot=false -x test
-    - uses: github/codeql-action/analyze@v1
+    - run: ./gradlew clean assemble -Dbuild.snapshot=false
+    - uses: github/codeql-action/analyze@v2
 
   build-artifact-names:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-java@v3
       with:

.github/workflows/code-hygiene.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: gradle/gradle-build-action@v2
         with:
           cache-disabled: true
-          arguments: checkstyleMain checkstyleTest
+          arguments: checkstyleMain checkstyleTest checkstyleIntegrationTest
 
   spotbugs:
     runs-on: ubuntu-latest

build.gradle

@@ -434,6 +434,14 @@ configurations {
             force "com.github.luben:zstd-jni:${versions.zstd}"
             force "org.xerial.snappy:snappy-java:1.1.10.4"
             force "com.google.guava:guava:${guava_version}"
+
+            // TODO: Seems like this should be removable
+            force "org.apache.httpcomponents:httpclient-cache:4.5.13"
+            force "org.apache.httpcomponents:httpclient:4.5.13"
+            force "org.apache.httpcomponents:fluent-hc:4.5.13"
+            force "org.apache.httpcomponents:httpcore:4.4.16"
+            force "org.apache.httpcomponents:httpcore-nio:4.4.16"
+            force "org.apache.httpcomponents:httpasyncclient:4.1.5"
         }
     }
 
@@ -449,6 +457,7 @@ sourceSets {
             srcDir file ('src/integrationTest/java')
             compileClasspath += sourceSets.main.output
             runtimeClasspath += sourceSets.main.output
+
         }
         resources {
             srcDir file('src/integrationTest/resources')
@@ -466,6 +475,7 @@ task integrationTest(type: Test) {
     systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
     classpath = sourceSets.integrationTest.runtimeClasspath
+    jvmArgs += "-Djdk.internal.httpclient.disableHostnameVerification"
     retry {
         failOnPassedAfterRetry = false
         maxRetries = 2
@@ -639,6 +649,11 @@ dependencies {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
     integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
+    integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.13"
+    integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.13"
+    integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.13"
+    integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.13"
+    integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
 
     //Checkstyle
     checkstyle 'com.puppycrawl.tools:checkstyle:10.12.1'

src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java

@@ -0,0 +1,29 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.common.logging;
+
+/**
+* Class uses to override OpenSearch NodeAndClusterIdConverter Log4j2 plugin in order to disable plugin and limit number of
+* warn messages like "...ApplierService#updateTask][T#1] WARN ClusterApplierService:628 - failed to notify ClusterStateListener..."
+* during tests execution.
+*
+* The class is rather a temporary solution and the real one should be developed in scope of:
+* https://github.com/opensearch-project/OpenSearch/pull/4322
+*/
+import org.apache.logging.log4j.core.LogEvent;
+
+class NodeAndClusterIdConverter {
+
+    public NodeAndClusterIdConverter() {}
+
+    public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) {}
+
+    public void format(LogEvent event, StringBuilder toAppendTo) {}
+}

src/integrationTest/java/org/opensearch/node/PluginAwareNode.java

@@ -0,0 +1,52 @@
+/*
+* Copyright 2015-2018 _floragunn_ GmbH
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*/
+
+package org.opensearch.node;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+
+public class PluginAwareNode extends Node {
+
+    private final boolean clusterManagerEligible;
+
+    @SafeVarargs
+    public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class<? extends Plugin>... plugins) {
+        super(
+            InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")),
+            Arrays.asList(plugins),
+            true
+        );
+        this.clusterManagerEligible = clusterManagerEligible;
+    }
+
+    public boolean isClusterManagerEligible() {
+        return clusterManagerEligible;
+    }
+}

src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java

@@ -0,0 +1,60 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.security;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Objects;
+
+class ConfigurationFiles {
+
+    public static void createRoleMappingFile(File destination) {
+        String resource = "roles_mapping.yml";
+        copyResourceToFile(resource, destination);
+    }
+
+    public static Path createConfigurationDirectory() {
+        try {
+            Path tempDirectory = Files.createTempDirectory("test-security-config");
+            String[] configurationFiles = {
+                "config.yml",
+                "action_groups.yml",
+                "config.yml",
+                "internal_users.yml",
+                "roles.yml",
+                "roles_mapping.yml",
+                "security_tenants.yml",
+                "tenants.yml" };
+            for (String fileName : configurationFiles) {
+                Path configFileDestination = tempDirectory.resolve(fileName);
+                copyResourceToFile(fileName, configFileDestination.toFile());
+            }
+            return tempDirectory.toAbsolutePath();
+        } catch (IOException ex) {
+            throw new RuntimeException("Cannot create directory with security plugin configuration.", ex);
+        }
+    }
+
+    private static void copyResourceToFile(String resource, File destination) {
+        try (InputStream input = ConfigurationFiles.class.getClassLoader().getResourceAsStream(resource)) {
+            Objects.requireNonNull(input, "Cannot find source resource " + resource);
+            try (OutputStream output = new FileOutputStream(destination)) {
+                input.transferTo(output);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Cannot create file with security plugin configuration", e);
+        }
+    }
+}