Refactor Gradle file for GeoSpatial build target

Signed-off-by: Andy Kwok <[email protected]> Remove redundent wrapper Signed-off-by: Andy Kwok <[email protected]> Github CI Signed-off-by: Andy Kwok <[email protected]> Update spotless Signed-off-by: Andy Kwok <[email protected]> Update gradle def Signed-off-by: Andy Kwok <[email protected]> Exclude geo IT from main integ-test Signed-off-by: Andy Kwok <[email protected]>
opensearch-project · Jan 15, 2025 · cf58dd4 · cf58dd4
1 parent e7be8ca
commit cf58dd4
Show file tree

Hide file tree

Showing 3 changed files with 263 additions and 82 deletions.
diff --git a/.github/workflows/integ-tests-with-geo.yml b/.github/workflows/integ-tests-with-geo.yml
@@ -0,0 +1,89 @@
+name: GeoSpatial Plugin IT
+
+on:
+  pull_request:
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+    paths:
+      - 'integ-test/**'
+      - '.github/workflows/integ-tests-with-geo.yml'
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  security-it-linux:
+    needs: Get-CI-Image-Tag
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [21]
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
+
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+
+    - name: Build with Gradle
+      run: |
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./gradlew integTestWithGeo"
+
+    - name: Upload test reports
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      continue-on-error: true
+      with:
+        name: test-reports-${{ matrix.os }}-${{ matrix.java }}
+        path: |
+          integ-test/build/reports/**
+          integ-test/build/testclusters/*/logs/*
+          integ-test/build/testclusters/*/config/*
+
+  security-it-windows-macos:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ windows-latest, macos-13 ]
+        java: [21]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+
+    - name: Build with Gradle
+      run: ./gradlew integTestWithGeo
+
+    - name: Upload test reports
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      continue-on-error: true
+      with:
+        name: test-reports-${{ matrix.os }}-${{ matrix.java }}
+        path: |
+          integ-test/build/reports/**
+          integ-test/build/testclusters/*/logs/*
+          integ-test/build/testclusters/*/config/*
diff --git a/integ-test/build.gradle b/integ-test/build.gradle
@@ -49,6 +49,12 @@ String bwcVersion = baseVersion + ".0";
 String baseName = "sqlBwcCluster"
 String bwcFilePath = "src/test/resources/bwc/"
 
+Map dummyLoginConfig = [
+        https: "false",
+        user: "admin",
+        password: "admin"
+]
+
 repositories {
     mavenCentral()
     maven { url 'https://jitpack.io' }
@@ -77,11 +83,24 @@ ext {
 
         return repo + "opensearch-security-${securitySnapshotVersion}.zip"
     }
-
     var projectAbsPath = projectDir.getAbsolutePath()
-    File downloadedSecurityPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-security-snapshot.zip').toFile()
+    configureGeoPlugin = { OpenSearchCluster cluster ->
+        File downloadedGeoPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-geospatial-snapshot.zip').toFile()
+        if (!downloadedGeoPlugin.exists()) {
+            download.run {
+                src getPluginDownloadLink("geospatial")
+                dest downloadedGeoPlugin
+            }
+        } else {
+            println "Geo-spatial Plugin File Already Exists"
+        }
+        cluster.plugin provider {(RegularFile) (() -> downloadedGeoPlugin)}
+    }
+
     configureSecurityPlugin = { OpenSearchCluster cluster ->
 
+        File downloadedSecurityPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-security-snapshot.zip').toFile()
+
         cluster.getNodes().forEach { node ->
             var creds = node.getCredentials()
             if (creds.isEmpty()) {
@@ -94,7 +113,7 @@ ext {
         // add a check to avoid re-downloading multiple times during single test run
         if (!downloadedSecurityPlugin.exists()) {
             download.run {
-                src getSecurityPluginDownloadLink()
+                src getPluginDownloadLink("opensearch-security")
                 dest downloadedSecurityPlugin
             }
         } else {
@@ -134,8 +153,7 @@ ext {
         ].forEach { name, value ->
             cluster.setting name, value
         }
-
-        cluster.plugin provider((Callable<RegularFile>) (() -> (RegularFile) (() -> downloadedSecurityPlugin)))
+        cluster.plugin provider {(RegularFile) (() -> downloadedSecurityPlugin)}
     }
 
     bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + baseVersion + '/latest/linux/x64/tar/builds/' +
@@ -224,21 +242,39 @@ testClusters.all {
 }
 
 def getJobSchedulerPlugin() {
-    provider(new Callable<RegularFile>() {
-        @Override
-        RegularFile call() throws Exception {
-            return new RegularFile() {
-                @Override
-                File getAsFile() {
-                    return configurations.zipArchive.asFileTree.matching {
-                        include '**/opensearch-job-scheduler*'
-                    }.singleFile
-                }
-            }
-        }
-    })
+    provider { (RegularFile) (() ->
+            configurations.zipArchive.asFileTree.matching {
+                include '**/opensearch-job-scheduler*'
+            }.singleFile )
+    }
+}
+
+static def getAllHttpSocketURI(cluster) {
+    return cluster.nodes.stream()
+            .flatMap { node -> node.getAllHttpSocketURI().stream() }
+            .collect(Collectors.joining(","))
 }
 
+static def getAllTransportSocketURI(cluster) {
+    return cluster.nodes.stream()
+            .flatMap { node -> node.getAllTransportPortURI().stream() }
+            .collect(Collectors.joining(","))
+}
+
+def getPluginDownloadLink(pluginName) {
+    var repo = "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/" +
+            pluginName + "/$opensearch_build_snapshot/"
+    var metadataFile = Paths.get(projectDir.toString(), "build", "maven-metadata.xml").toAbsolutePath().toFile()
+    download.run {
+        src repo + "maven-metadata.xml"
+        dest metadataFile
+    }
+    def metadata = new XmlParser().parse(metadataFile)
+    def PluginSnapshotVersion = metadata.versioning.snapshotVersions[0].snapshotVersion[0].value[0].text()
+    return repo + pluginName +"-${PluginSnapshotVersion}.zip"
+}
+
+
 testClusters {
     integTest {
         testDistribution = 'archive'
@@ -261,6 +297,11 @@ testClusters {
         plugin(getJobSchedulerPlugin())
         plugin ":opensearch-sql-plugin"
     }
+    integTestWithGeo {
+        testDistribution = 'archive'
+        plugin(getJobSchedulerPlugin())
+        plugin ":opensearch-sql-plugin"
+    }
 }
 
 task startPrometheus(type: SpawnProcessTask) {
@@ -370,21 +411,51 @@ task integTestWithSecurity(type: RestIntegTestTask) {
     doFirst {
         systemProperty 'cluster.debug', getDebug()
         getClusters().forEach { cluster ->
+            systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> getAllHttpSocketURI(cluster)}"
+            systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> getAllTransportSocketURI(cluster)}"
+        }
+        systemProperties dummyLoginConfig
+    }
 
-            String allTransportSocketURI = cluster.nodes.stream().flatMap { node ->
-                node.getAllTransportPortURI().stream()
-            }.collect(Collectors.joining(","))
-            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
-                node.getAllHttpSocketURI().stream()
-            }.collect(Collectors.joining(","))
+    if (System.getProperty("test.debug") != null) {
+        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+    }
+
+    // NOTE: this IT config discovers only junit5 (jupiter) tests.
+    // https://github.com/opensearch-project/sql/issues/1974
+    filter {
+        includeTestsMatching 'org.opensearch.sql.security.CrossClusterSearchIT'
+    }
+}
 
-            systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
-            systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> allTransportSocketURI}"
-        }
 
-        systemProperty "https", "false"
-        systemProperty "user", "admin"
-        systemProperty "password", "admin"
+task integTestWithGeo(type: RestIntegTestTask) {
+    useCluster testClusters.remoteCluster
+
+    getClusters().forEach { cluster ->
+        configureGeoPlugin(cluster)
+    }
+
+    useJUnitPlatform()
+    dependsOn ':opensearch-sql-plugin:bundlePlugin'
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+
+    systemProperty 'tests.security.manager', 'false'
+    systemProperty 'project.root', project.projectDir.absolutePath
+    // Set default query size limit
+    systemProperty 'defaultQuerySizeLimit', '10000'
+
+    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
+    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
+    doFirst {
+        systemProperty 'cluster.debug', getDebug()
+        getClusters().forEach { cluster ->
+            systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> getAllHttpSocketURI(cluster)}"
+            systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> getAllTransportSocketURI(cluster)}"
+        }
+        systemProperties dummyLoginConfig
     }
 
     if (System.getProperty("test.debug") != null) {
@@ -394,26 +465,20 @@ task integTestWithSecurity(type: RestIntegTestTask) {
     // NOTE: this IT config discovers only junit5 (jupiter) tests.
     // https://github.com/opensearch-project/sql/issues/1974
     filter {
-        includeTestsMatching 'org.opensearch.sql.security.CrossClusterSearchIT'
+        includeTestsMatching 'org.opensearch.sql.geo.PplIpEnrichmentIT'
     }
 }
 
+
 // Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled
 integTest {
     useCluster testClusters.remoteCluster
 
     // Set properties for connection to clusters and between clusters
     doFirst {
         getClusters().forEach { cluster ->
-            String allTransportSocketURI = cluster.nodes.stream().flatMap { node ->
-                node.getAllTransportPortURI().stream()
-            }.collect(Collectors.joining(","))
-            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
-                node.getAllHttpSocketURI().stream()
-            }.collect(Collectors.joining(","))
-
-            systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
-            systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> allTransportSocketURI}"
+            systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> getAllHttpSocketURI(cluster)}"
+            systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> getAllTransportSocketURI(cluster)}"
         }
     }
 
@@ -480,8 +545,9 @@ integTest {
     // Exclude JDBC related tests
     exclude 'org/opensearch/sql/jdbc/**'
 
-    // Exclude this IT, because they executed in another task (:integTestWithSecurity)
+    // Exclude these IT, because they executed in respective plugin-related tasks (ex: :integTestWithSecurity)
     exclude 'org/opensearch/sql/security/**'
+    exclude 'org/opensearch/sql/geo/**'
 }
 
 
@@ -526,37 +592,22 @@ task comparisonTest(type: RestIntegTestTask) {
             testDistribution = "ARCHIVE"
             versions = [baseVersion, opensearch_version]
             numberOfNodes = 3
-            plugin(provider(new Callable<RegularFile>(){
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
-                                project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
-                            }
-                            project.mkdir bwcJobSchedulerPath + bwcVersion
-                            ant.get(src: bwcOpenSearchJSDownload,
-                                    dest: bwcJobSchedulerPath + bwcVersion,
-                                    httpusecaches: false)
-                            return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
-                        }
+            plugin(provider { (RegularFile) ({
+                    if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
+                        project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
                     }
-                }
-            }))
-            plugin(provider(new Callable<RegularFile>(){
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            return configurations.zipArchive.asFileTree.matching {
-                                include '**/opensearch-sql-plugin*'
-                            }.singleFile
-                        }
-                    }
-                }
-            }))
+                    project.mkdir bwcJobSchedulerPath + bwcVersion
+                    ant.get(src: bwcOpenSearchJSDownload,
+                            dest: bwcJobSchedulerPath + bwcVersion,
+                            httpusecaches: false)
+                    fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
+                })
+            })
+            plugin(provider { (RegularFile) (
+                configurations.zipArchive.asFileTree.matching {
+                    include '**/opensearch-sql-plugin*'
+                }.singleFile )
+            })
             setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
             setting 'http.content_type.required', 'true'
         }
@@ -565,17 +616,9 @@ task comparisonTest(type: RestIntegTestTask) {
 
 List<Provider<RegularFile>> plugins = [
         getJobSchedulerPlugin(),
-        provider(new Callable<RegularFile>() {
-            @Override
-            RegularFile call() throws Exception {
-                return new RegularFile() {
-                    @Override
-                    File getAsFile() {
-                        return fileTree(bwcFilePath + project.version).getSingleFile()
-                    }
-                }
-            }
-        })
+        provider{ (RegularFile) (
+                fileTree(bwcFilePath + project.version).getSingleFile())
+        }
 ]
 
 // Creates 2 test clusters with 3 nodes of the old version.