From 4e6334557b962c2e3798133617f6dadb1dc09719 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 19 Jan 2024 14:24:09 -0800 Subject: [PATCH 1/8] Fixing security plugin archive Signed-off-by: Varun Jain --- build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index c4228d8be..59d94a7f4 100644 --- a/build.gradle +++ b/build.gradle @@ -66,7 +66,7 @@ ext { projectSubstitutions = [:] configureSecurityPlugin = { OpenSearchCluster cluster -> - configurations.zipArchive.asFileTree.each { + configurations.secureIntegTestPluginArchive.asFileTree.each { if(it.name.contains("opensearch-security")){ cluster.plugin(provider(new Callable() { @Override @@ -160,6 +160,7 @@ allprojects { configurations { zipArchive + secureIntegTestPluginArchive } tasks.register("preparePluginPathDirs") { @@ -231,7 +232,7 @@ dependencies { api "org.opensearch:opensearch:${opensearch_version}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" - zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" + secureIntegTestPluginArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" From 9fffb17a552f7a4cab869f267ade0aa167dc2f16 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 19 Jan 2024 15:21:29 -0800 Subject: [PATCH 2/8] Disabling bwc tests and security tests Signed-off-by: Varun Jain --- .../workflows/backwards_compatibility_tests_workflow.yml | 2 -- .github/workflows/test_security.yml | 2 +- build.gradle | 6 ++---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 32a71c5d0..539c14ebc 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -35,7 +35,6 @@ jobs: - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' Rolling-Upgrade-BWCTests-NeuralSearch: strategy: @@ -62,4 +61,3 @@ jobs: - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 5b7f9d210..009e18ca9 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -47,4 +47,4 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" + su `id -un 1000` -c "whoami && java -version" diff --git a/build.gradle b/build.gradle index 59d94a7f4..c2dc3a317 100644 --- a/build.gradle +++ b/build.gradle @@ -338,8 +338,7 @@ testClusters.integTest { // Install K-NN/ml-commons plugins on the integTest cluster nodes except security configurations.zipArchive.asFileTree.each { - if(!it.name.contains("opensearch-security")) { - plugin(provider(new Callable(){ + plugin(provider(new Callable(){ @Override RegularFile call() throws Exception { return new RegularFile() { @@ -349,8 +348,7 @@ testClusters.integTest { } } } - })) - } + })) } // This installs our neural-search plugin into the testClusters From 0ca308a8493912709821105a457b90c744f7060d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 19 Jan 2024 15:48:39 -0800 Subject: [PATCH 3/8] Increase socket timeout Signed-off-by: Varun Jain --- .../opensearch/neuralsearch/OpenSearchSecureRestTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index 133f42daf..2d7cc5cbe 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -61,7 +61,7 @@ public abstract class OpenSearchSecureRestTestCase extends OpenSearchRestTestCas private static final String SYS_PROPERTY_KEY_CLUSTER_ENDPOINT = "tests.rest.cluster"; private static final String SYS_PROPERTY_KEY_USER = "user"; private static final String SYS_PROPERTY_KEY_PASSWORD = "password"; - private static final String DEFAULT_SOCKET_TIMEOUT = "60s"; + private static final String DEFAULT_SOCKET_TIMEOUT = "120s"; private static final String INTERNAL_INDICES_PREFIX = "."; private static String protocol; From 470d519c13f5ff76a4ea09c39c35679f6567e501 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 19 Jan 2024 16:01:06 -0800 Subject: [PATCH 4/8] Increase Timeout Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java | 2 +- .../opensearch/neuralsearch/OpenSearchSecureRestTestCase.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 680d90b65..23c353d73 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -63,7 +63,7 @@ public abstract class BaseNeuralSearchIT extends OpenSearchSecureRestTestCase { protected static final Locale LOCALE = Locale.ROOT; - private static final int MAX_TASK_RESULT_QUERY_TIME_IN_SECOND = 60 * 5; + private static final int MAX_TASK_RESULT_QUERY_TIME_IN_SECOND = 90 * 5; private static final int DEFAULT_TASK_RESULT_QUERY_INTERVAL_IN_MILLISECOND = 1000; protected static final String DEFAULT_USER_AGENT = "Kibana"; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index 2d7cc5cbe..133f42daf 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -61,7 +61,7 @@ public abstract class OpenSearchSecureRestTestCase extends OpenSearchRestTestCas private static final String SYS_PROPERTY_KEY_CLUSTER_ENDPOINT = "tests.rest.cluster"; private static final String SYS_PROPERTY_KEY_USER = "user"; private static final String SYS_PROPERTY_KEY_PASSWORD = "password"; - private static final String DEFAULT_SOCKET_TIMEOUT = "120s"; + private static final String DEFAULT_SOCKET_TIMEOUT = "60s"; private static final String INTERNAL_INDICES_PREFIX = "."; private static String protocol; From 26fcf06b268b795cb42e4aa25ceab3a6475da56f Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 19 Jan 2024 16:39:22 -0800 Subject: [PATCH 5/8] Reverting the timout change Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 23c353d73..680d90b65 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -63,7 +63,7 @@ public abstract class BaseNeuralSearchIT extends OpenSearchSecureRestTestCase { protected static final Locale LOCALE = Locale.ROOT; - private static final int MAX_TASK_RESULT_QUERY_TIME_IN_SECOND = 90 * 5; + private static final int MAX_TASK_RESULT_QUERY_TIME_IN_SECOND = 60 * 5; private static final int DEFAULT_TASK_RESULT_QUERY_INTERVAL_IN_MILLISECOND = 1000; protected static final String DEFAULT_USER_AGENT = "Kibana"; From 322b4db8a322fcfca97278b9e10df3c62f4b6ef6 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 22 Jan 2024 15:42:38 -0800 Subject: [PATCH 6/8] Specifying java 21 version Signed-off-by: Varun Jain --- .github/workflows/CI.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e3f749fac..0f5e7f8ff 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,7 +21,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21.0.1] os: [ubuntu-latest] name: Gradle Check Linux @@ -33,6 +33,7 @@ jobs: # need to switch to root so that github actions can install runner binary on container without permission issues. options: --user root + steps: - name: Checkout neural-search uses: actions/checkout@v1 @@ -55,7 +56,7 @@ jobs: Check-neural-search-windows: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21.0.1] os: [windows-latest] name: Gradle Check Windows @@ -83,7 +84,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21.0.1] os: [ubuntu-latest] name: Pre-commit Linux @@ -117,7 +118,7 @@ jobs: Precommit-neural-search-windows: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21.0.1] os: [windows-latest] name: Pre-commit Windows From 6874cc5fa5b644fbb12a1f207d539e95d9e8d3ed Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 22 Jan 2024 15:54:10 -0800 Subject: [PATCH 7/8] Adding comment to java version restriction Signed-off-by: Varun Jain --- .github/workflows/CI.yml | 4 ++++ .github/workflows/backwards_compatibility_tests_workflow.yml | 5 ++++- .github/workflows/test_security.yml | 5 +++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0f5e7f8ff..3f89a3185 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,6 +21,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: + # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted. java: [11, 17, 21.0.1] os: [ubuntu-latest] @@ -56,6 +57,7 @@ jobs: Check-neural-search-windows: strategy: matrix: + # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted. java: [11, 17, 21.0.1] os: [windows-latest] @@ -84,6 +86,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: + # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted. java: [11, 17, 21.0.1] os: [ubuntu-latest] @@ -118,6 +121,7 @@ jobs: Precommit-neural-search-windows: strategy: matrix: + # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted. java: [11, 17, 21.0.1] os: [windows-latest] diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 539c14ebc..030f930fa 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -13,7 +13,8 @@ jobs: Restart-Upgrade-BWCTests-NeuralSearch: strategy: matrix: - java: [ 11, 17, 21 ] + # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted. + java: [ 11, 17, 21.0.1 ] os: [ubuntu-latest,windows-latest] bwc_version : ["2.9.0","2.10.0","2.11.0","2.12.0-SNAPSHOT"] opensearch_version : [ "3.0.0-SNAPSHOT" ] @@ -35,6 +36,7 @@ jobs: - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running restart-upgrade backwards compatibility tests ..." +# ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' Rolling-Upgrade-BWCTests-NeuralSearch: strategy: @@ -61,3 +63,4 @@ jobs: - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running rolling-upgrade backwards compatibility tests ..." +# ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 009e18ca9..163b8c220 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -20,7 +20,8 @@ jobs: integ-test-with-security-linux: strategy: matrix: - java: [11, 17, 21] + # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted. + java: [11, 17, 21.0.1] name: Run Integration Tests on Linux runs-on: ubuntu-latest @@ -47,4 +48,4 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version" + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" From d5dd7dcc89170e43b4d28bee8f0f52d6ba4a3d78 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 22 Jan 2024 16:15:10 -0800 Subject: [PATCH 8/8] Adding comment for disabling bwc tests Signed-off-by: Varun Jain --- .github/workflows/backwards_compatibility_tests_workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 030f930fa..9b34b6356 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -36,6 +36,7 @@ jobs: - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running restart-upgrade backwards compatibility tests ..." +# Disabling BWC tests due to ongoing build failure. https://github.com/opensearch-project/neural-search/issues/536 # ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' Rolling-Upgrade-BWCTests-NeuralSearch: @@ -63,4 +64,5 @@ jobs: - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running rolling-upgrade backwards compatibility tests ..." +# Disabling BWC tests due to ongoing build failure. https://github.com/opensearch-project/neural-search/issues/536 # ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'