add fix for classpath change and for cmake discovery on macos

Signed-off-by: Samuel Herman <[email protected]>
opensearch-project · Feb 7, 2025 · dd9a595 · dd9a595
1 parent 5d61402
commit dd9a595
Show file tree

Hide file tree

Showing 16 changed files with 95 additions and 38 deletions.
diff --git a/build.gradle b/build.gradle
@@ -329,31 +329,88 @@ task windowsPatches(type:Exec) {
     commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1"
 }
 
-task cmakeJniLib(type:Exec) {
-    workingDir 'jni'
-    def args = []
-    args.add("cmake")
-    args.add("-B build")
-    args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}")
-    args.add("-DAVX2_ENABLED=${avx2_enabled}")
-    args.add("-DAVX512_ENABLED=${avx512_enabled}")
-    args.add("-DAVX512_SPR_ENABLED=${avx512_spr_enabled}")
-    args.add("-DCOMMIT_LIB_PATCHES=${commit_lib_patches}")
-    args.add("-DAPPLY_LIB_PATCHES=${apply_lib_patches}")
-    def javaHome = Jvm.current().getJavaHome()
-    logger.lifecycle("Java home directory used by gradle: $javaHome")
-    if (Os.isFamily(Os.FAMILY_MAC)) {
-        environment('JAVA_HOME',javaHome)
-    }
-    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-        dependsOn windowsPatches
-        args.add("-G")
-        args.add("Unix Makefiles")
-        args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
-        args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
+tasks.register('checkCmakePath', Exec) {
+    def outputStream = new ByteArrayOutputStream()
+    def findCmakePathArgs = []
+    findCmakePathArgs.add("which")
+    findCmakePathArgs.add("cmake")
+    commandLine findCmakePathArgs
+    standardOutput = outputStream
+    doLast {
+        def cmakePath = outputStream.toString().trim()
+        println "CMake path: ${cmakePath}"
+        // Ensure cmakePath is treated as a String. If parsing to an int is required,
+        // handle it explicitly, though a path typically should not be parsed as an int.
+        if (cmakePath.isEmpty()) {
+            throw new GradleException("CMake not found in PATH. Please install CMake.")
+        }
+        workingDir 'jni'
+        def args = []
+        args.add("cmake")
+        args.add("-B build")
+        args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}")
+        args.add("-DAVX2_ENABLED=${avx2_enabled}")
+        args.add("-DAVX512_ENABLED=${avx512_enabled}")
+        args.add("-DAVX512_SPR_ENABLED=${avx512_spr_enabled}")
+        args.add("-DCOMMIT_LIB_PATCHES=${commit_lib_patches}")
+        args.add("-DAPPLY_LIB_PATCHES=${apply_lib_patches}")
+        def javaHome = Jvm.current().getJavaHome()
+        logger.lifecycle("Java home directory used by gradle: $javaHome")
+        if (Os.isFamily(Os.FAMILY_MAC)) {
+            environment('JAVA_HOME', javaHome)
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            dependsOn windowsPatches
+            args.add("-G")
+            args.add("Unix Makefiles")
+            args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
+            args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
+        }
+
+        commandLine args
     }
+}
 
-    commandLine args
+tasks.register('cmakeJniLib', Exec) {
+    def outputStream = new ByteArrayOutputStream()
+    def findCmakePathArgs = []
+    findCmakePathArgs.add("which")
+    findCmakePathArgs.add("cmake")
+    commandLine findCmakePathArgs
+    standardOutput = outputStream
+    doLast {
+        def cmakePath = outputStream.toString().trim()
+        println "CMake path: ${cmakePath}"
+        // Ensure cmakePath is treated as a String. If parsing to an int is required,
+        // handle it explicitly, though a path typically should not be parsed as an int.
+        if (cmakePath.isEmpty()) {
+            throw new GradleException("CMake not found in PATH. Please install CMake.")
+        }
+        workingDir 'jni'
+        def args = []
+        args.add("cmake")
+        args.add("-B build")
+        args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}")
+        args.add("-DAVX2_ENABLED=${avx2_enabled}")
+        args.add("-DAVX512_ENABLED=${avx512_enabled}")
+        args.add("-DAVX512_SPR_ENABLED=${avx512_spr_enabled}")
+        args.add("-DCOMMIT_LIB_PATCHES=${commit_lib_patches}")
+        args.add("-DAPPLY_LIB_PATCHES=${apply_lib_patches}")
+        def javaHome = Jvm.current().getJavaHome()
+        logger.lifecycle("Java home directory used by gradle: $javaHome")
+        if (Os.isFamily(Os.FAMILY_MAC)) {
+            environment('JAVA_HOME', javaHome)
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            dependsOn windowsPatches
+            args.add("-G")
+            args.add("Unix Makefiles")
+            args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
+            args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
+        }
+
+        commandLine args
+    }
 }
 
 task buildJniLib(type:Exec) {

diff --git a/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java b/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java
@@ -13,7 +13,7 @@
 import org.opensearch.knn.plugin.transport.KNNStatsResponse;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.threadpool.ThreadPool;

diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java
@@ -11,7 +11,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Booleans;

diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java
@@ -36,7 +36,7 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.health.ClusterIndexHealth;
 import org.opensearch.cluster.metadata.IndexMetadata;

diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java
@@ -49,7 +49,7 @@
 import com.google.common.collect.ImmutableList;
 
 import org.opensearch.action.ActionRequest;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java
@@ -8,7 +8,7 @@
 import com.google.common.collect.ImmutableList;
 import lombok.AllArgsConstructor;
 import lombok.extern.log4j.Log4j2;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.core.common.Strings;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java
@@ -12,7 +12,7 @@
 package org.opensearch.knn.plugin.rest;
 
 import com.google.common.collect.ImmutableList;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.core.common.Strings;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.transport.DeleteModelAction;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java
@@ -13,7 +13,7 @@
 
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.lang.StringUtils;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.transport.GetModelAction;
 import org.opensearch.knn.plugin.transport.GetModelRequest;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java
@@ -11,7 +11,7 @@
 import org.opensearch.knn.plugin.transport.KNNStatsAction;
 import org.opensearch.knn.plugin.transport.KNNStatsRequest;
 import com.google.common.collect.ImmutableList;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.RestActions;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java
@@ -13,7 +13,7 @@
 import com.google.common.collect.ImmutableList;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestSearchModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestSearchModelHandler.java
@@ -13,7 +13,7 @@
 
 import com.google.common.collect.ImmutableList;
 import org.opensearch.action.search.SearchRequest;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.transport.SearchModelAction;
 import org.opensearch.rest.BaseRestHandler;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java
@@ -12,7 +12,7 @@
 package org.opensearch.knn.plugin.rest;
 
 import com.google.common.collect.ImmutableList;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.mapper.NumberFieldMapper;
 import org.opensearch.knn.common.KNNConstants;

diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java
@@ -17,7 +17,7 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.HandledTransportAction;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.ValidationException;

diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java
@@ -17,7 +17,7 @@
 import org.opensearch.action.search.SearchRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchScrollRequestBuilder;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.ValidationException;

diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java
@@ -16,7 +16,7 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.ActionFilters;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.node.DiscoveryNode;

diff --git a/...org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java b/...org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java
@@ -9,7 +9,7 @@
 import lombok.SneakyThrows;
 import org.junit.After;
 import org.junit.Before;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;