Merge pull request #62 from NvTimLiu/release-tmp

Change version and dependency jars to v22.02.0

.github/workflows/auto-merge.yml

@@ -0,0 +1,41 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to keep BASE branch up-to-date from HEAD branch
+name: auto-merge HEAD to BASE
+
+on:
+  pull_request_target:
+    branches:
+    - branch-22.02
+    types: [closed]
+
+jobs:
+  auto-merge:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: branch-22.02 # force to fetch from latest upstream instead of PR ref
+
+      - name: auto-merge job
+        uses: ./.github/workflows/auto-merge
+        env:
+          OWNER: NVIDIA
+          REPO_NAME: spark-rapids-ml
+          HEAD: branch-22.02
+          BASE: branch-22.04
+          AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR

.github/workflows/auto-merge/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:alpine
+
+WORKDIR /
+COPY automerge .
+RUN pip install requests && chmod +x /automerge
+
+# require envs: OWNER,REPO_NAME,HEAD,BASE,GITHUB_TOKEN
+ENTRYPOINT ["/automerge"]

.github/workflows/auto-merge/action.yml

@@ -0,0 +1,20 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'auto-merge action'
+description: 'auto-merge HEAD to BASE'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+

.github/workflows/auto-merge/automerge

@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A auto-merge tool
+
+Create a PR to merge HEAD to BASE branch.
+NOTE:
+    The generated PR should be automatically merged if no conflict. Otherwise, manual operation will be required.
+"""
+
+import os
+import sys
+import time
+
+import requests
+
+# ENV
+OWNER = os.environ.get('OWNER')
+assert OWNER, 'env OWNER should not be empty'
+REPO_NAME = os.environ.get('REPO_NAME')
+assert REPO_NAME, 'env REPO_NAME should not be empty'
+HEAD = os.environ.get('HEAD')
+assert HEAD, 'env HEAD should not be empty'
+BASE = os.environ.get('BASE')
+assert BASE, 'env BASE should not be empty'
+AUTOMERGE_TOKEN = os.environ.get('AUTOMERGE_TOKEN')
+assert AUTOMERGE_TOKEN, 'env AUTOMERGE_TOKEN should not be empty'
+# static
+API_URL = 'https://api.github.com'
+AUTH_HEADERS = {
+    'Authorization': 'token ' + AUTOMERGE_TOKEN
+}
+
+
+def create():
+    url = f'{API_URL}/repos/{OWNER}/{REPO_NAME}/pulls'
+    params = {
+        'title': f'[auto-merge] {HEAD} to {BASE} [skip ci] [bot]',
+        'head': HEAD,
+        'base': BASE,
+        'body': f'auto-merge triggered by github actions on `{HEAD}` to create a PR keeping `{BASE}` up-to-date. If '
+                'this PR is unable to be merged due to conflicts, it will remain open until manually fix.',
+        'maintainer_can_modify': True
+    }
+    r = requests.post(url, headers=AUTH_HEADERS, json=params)
+    if r.status_code == 201:
+        print('SUCCESS - create PR')
+        pull = r.json()
+        number = str(pull['number'])
+        sha = str(pull['head']['sha'])
+        return number, sha, False
+    if r.status_code == 422:  # early-terminate if no commits between HEAD and BASE
+        print('SUCCESS - No commits')
+        print(r.json())
+        return '', '', True
+    # FAILURE
+    print('FAILURE - create PR')
+    print(f'status code: {r.status_code}')
+    print(r.json())
+    sys.exit(1)
+
+
+def auto_merge(number, sha):
+    url = f'{API_URL}/repos/{OWNER}/{REPO_NAME}/pulls/{number}/merge'
+    params = {
+        'sha': sha,
+        'merge_method': 'merge'
+    }
+    r = requests.put(url, headers=AUTH_HEADERS, json=params)
+    if r.status_code == 200:
+        comment(number, '**SUCCESS** - auto-merge')
+        print('SUCCESS - auto-merge')
+        sys.exit(0)
+    else:
+        print('FAILURE - auto-merge')
+        comment(number=number, content=f"""**FAILURE** - Unable to auto-merge. Manual operation is required.
+```
+{r.json()}
+```
+
+Please use the following steps to fix the merge conflicts manually:
+```
+# Assume upstream is NVIDIA/spark-rapids-ml remote
+git fetch upstream {HEAD} {BASE}
+git checkout -b fix-auto-merge-conflict-{number} upstream/{BASE}
+git merge upstream/{HEAD}
+# Fix any merge conflicts caused by this merge
+git commit -am "Merge {HEAD} into {BASE}"
+git push <personal fork> fix-auto-merge-conflict-{number}
+# Open a PR targets NVIDIA/spark-rapids-ml {BASE}
+```
+**IMPORTANT:** Before merging this PR, be sure to change the merging strategy to `Create a merge commit` (repo admin only).
+
+Once this PR is merged, the auto-merge PR should automatically be closed since it contains the same commit hashes
+""")
+        print(f'status code: {r.status_code}')
+        print(r.json())
+        sys.exit(1)
+
+
+def comment(number, content):
+    url = f'{API_URL}/repos/{OWNER}/{REPO_NAME}/issues/{number}/comments'
+    params = {
+        'body': content
+    }
+    r = requests.post(url, headers=AUTH_HEADERS, json=params)
+    if r.status_code == 201:
+        print('SUCCESS - create comment')
+    else:
+        print('FAILURE - create comment')
+        print(f'status code: {r.status_code}')
+        print(r.json())
+
+
+def main():
+    number, sha, term = create()
+    if term:
+        sys.exit(0)
+
+    auto_merge(number, sha)
+
+
+if __name__ == '__main__':
+    main()

README.md

@@ -25,23 +25,16 @@ We used a customized class and user will need to do `no code change` to enjoy th
 
 ```scala
 val pca = new com.nvidia.spark.ml.feature.PCA()
+  .setInputCol("feature_array_type") // accept ArrayType column, no need to convert it to Vector type
+  .setOutputCol("feature_value_3d")
+  .setK(3)
+  .fit(vectorDf)
 ...
 ```
 
-Besides, we provide some switch APIs to allow users to highly customize their training process:
-
-```scala
-  .useGemm(true) // or false, default: true. Switch to use original BLAS bsr or cuBLAS gemm to compute covariance matrix
-  .useCuSolverSVD(true) // or false, default: true. Switch to use original LAPack solver or cuSolver to compute SVD
-  .meanCentering(true) // or false, default: true. Switch to do mean centering or not before computing covariance matrix
-```
-
-To speedup the transform process, it's required to add an extra setting:
-```scala
-  .setTransformInputCol("feature_array_type")
-```
-Note: The `setInputCol` is targeting the input column of `Vector` type for training process, while
- the `setTransformInputCol` is for column of ArrayType.
+Note: The `setInputCol` is targeting the input column of `Vector` type for training process in `CPU`
+version. But in GPU version, user doesn't need to do the extra preprocess step to convert column of
+`ArrayType` to `Vector` type, the `setInputCol` will accept the raw `ArrayType` column.
 
 ## Build
 
@@ -56,9 +49,9 @@ and cmake dependecies
 4. [cuDF](https://github.com/rapidsai/cudf):
     - install cuDF shared library via conda:
       ```bash
-      conda install -c rapidsai-nightly -c nvidia -c conda-forge cudf=21.12 python=3.8 -y
+      conda install -c rapidsai-nightly -c nvidia -c conda-forge cudf=22.02 python=3.8 -y
       ```
-5. [RAFT(21.12)](https://github.com/rapidsai/raft):
+5. [RAFT(22.02)](https://github.com/rapidsai/raft):
     - raft provides only header files, so no build instructions for it.
       ```bash
       $ git clone -b branch-21.12 https://github.com/rapidsai/raft.git
@@ -72,7 +65,7 @@ User can build it directly in the _project root path_ with:
 ```
 mvn clean package
 ```
-Then `rapids-4-spark-ml_2.12-21.12.0.jar` will be generated under `target` folder.
+Then `rapids-4-spark-ml_2.12-22.02.0.jar` will be generated under `target` folder.
 
 _Note_: This module contains both native and Java/Scala code. The native library build instructions
 has been added to the pom.xml file so that maven build command will help build native library all
@@ -85,9 +78,9 @@ repository, usually in your `$HOME/.m2/repository`.
 
 Add the artifact jar to the Spark, for example:
 ```bash
-ML_JAR="target/rapids-4-spark-ml_2.12-21.12.0.jar"
-CUDF_JAR="$HOME/.m2/repository/ai/rapids/cudf/21.12.0/cudf-21.12.0.jar"
-PLUGIN_JAR="$HOME/.m2/repository/com/nvidia/rapids-4-spark_2.12/21.12.0/rapids-4-spark_2.12-21.12.0.jar"
+ML_JAR="target/rapids-4-spark-ml_2.12-22.02.0.jar"
+CUDF_JAR="~/.m2/repository/ai/rapids/cudf/22.02.0/cudf-22.02.0.jar"
+PLUGIN_JAR="~/.m2/repository/com/nvidia/rapids-4-spark_2.12/22.02.0/rapids-4-spark_2.12-22.02.0.jar"
 
 $SPARK_HOME/bin/spark-shell --master $SPARK_MASTER \
  --driver-memory 20G \
@@ -104,9 +97,9 @@ $SPARK_HOME/bin/spark-shell --master $SPARK_MASTER \
 ### PCA examples
 
 Please refer to
-[PCA examples](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-21.12/examples/Spark-cuML/pca/) for
+[PCA examples](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-22.02/examples/Spark-cuML/pca/) for
 more details about example code. We provide both
-[Notebook](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-21.12/examples/Spark-cuML/pca/PCA-example-notebook.ipynb)
-and [jar](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-21.12/examples/Spark-cuML/pca/scala/src/com/nvidia/spark/examples/pca/Main.scala)
+[Notebook](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-22.02/examples/Spark-cuML/pca/PCA-example-notebook.ipynb)
+and [jar](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-22.02/examples/Spark-cuML/pca/scala/src/com/nvidia/spark/examples/pca/Main.scala)
  versions there. Instructions to run these examples are described in the
- [README](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-21.12/examples/Spark-cuML/pca/README.md).
+ [README](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-22.02/examples/Spark-cuML/pca/README.md).

native/src/rapidsml_jni.cpp

@@ -19,6 +19,7 @@
 #include <iostream>
 #include <stdlib.h>
 
+#include <cudf/binaryop.hpp>
 #include <cudf/column/column.hpp>
 #include <cudf/column/column_view.hpp>
 #include <cudf/detail/sequence.hpp>
@@ -33,13 +34,24 @@ extern "C" {
 
 JNIEXPORT jlong JNICALL Java_com_nvidia_spark_ml_linalg_JniRAPIDSML_dgemmWithColumnViewPtr(
   JNIEnv *env, jclass, jint transa, jint transb, jint m, jint n, jint k, jdouble alpha,
-  jdoubleArray A, jint lda, jlong B,jint ldb, jdouble beta, jint ldc, jint deviceID)
-{
+  jdoubleArray A, jint lda, jlong B,jint ldb, jdouble beta, jint ldc, jint deviceID) {
   try {
     cudf::jni::native_jdoubleArray native_A(env, A);
-    auto ret_column = dgemm(transa, transb, m, n, k, alpha, native_A.data(), native_A.size(), lda, B, ldb, beta, ldc, deviceID);
+    auto ret_column = dgemm(transa, transb, m, n, k, alpha, native_A.data(), native_A.size(), lda,
+                            B, ldb, beta, ldc, deviceID);
     return ret_column;
   }
   CATCH_STD(env, 0);
 }
+
+JNIEXPORT jlong JNICALL Java_com_nvidia_spark_ml_linalg_JniRAPIDSML_dgemmCov(JNIEnv *env, jclass,
+  jint transa, jint transb, jint m, jint n, jint k, jdouble alpha, jlong A, jint lda, jlong B,
+  jint ldb, jdouble beta, jdoubleArray C, jint ldc, jint deviceID) {
+  try {
+    cudf::jni::native_jdoubleArray native_C(env, C);
+    dgemmCov(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, native_C.data(), ldc, deviceID);
+  }
+  CATCH_STD(env, 0);
+}
+
 }  // extern "C"

native/src/rapidsml_jni.cu

@@ -107,6 +107,25 @@ long dgemm(int transa, int transb, int m, int n,int k, double alpha, double* A,
     return reinterpret_cast<long>(target_column.release());
 }
 
+void dgemmCov(int transa, int transb, int m, int n,int k, double alpha, long A, int lda,long B,
+              int ldb, double beta, double* C, int ldc, int deviceID) {
+  cudaSetDevice(deviceID);
+  raft::handle_t raft_handle;
+  cudaStream_t stream = raft_handle.get_stream();
+  auto const *cv_ptr = reinterpret_cast<cudf::lists_column_view const *>(A);
+  auto const child_column_view = cv_ptr->child();
+  auto c_stream = rmm::cuda_stream_view(stream);
+  auto size_C = m * n;
+  // create child column that will own the computation result
+  rmm::device_buffer dev_buff_C = rmm::device_buffer(C, size_C * sizeof(double), c_stream);
+  auto status = raft::linalg::cublasgemm(raft_handle.get_cublas_handle(),
+                                         convertToCublasOpEnum(transa),
+                                         convertToCublasOpEnum(transb),
+                                         m, n, k, &alpha, child_column_view.data<double>(), lda,
+                                         child_column_view.data<double>(),ldb, &beta,
+                                         (double*)dev_buff_C.data(), ldc, stream);
+  cudaMemcpyAsync(C, dev_buff_C.data(), size_C * sizeof(double), cudaMemcpyDefault);
+}
 
 extern "C" {
 
@@ -249,5 +268,4 @@ JNIEXPORT void JNICALL Java_com_nvidia_spark_ml_linalg_JniRAPIDSML_calSVD
     env->ReleaseDoubleArrayElements(U, host_U, 0);
     env->ReleaseDoubleArrayElements(S, host_S, 0);
   }
-
 }// extern "C"

native/src/rapidsml_jni.hpp

@@ -26,3 +26,6 @@ void signFlip(double* input, int n_rows, int n_cols, double* components,
 long dgemm(int transa, int transb, int m, int n,
            int k, double alpha, double* A, int size_A, int lda, long B,
            int ldb, double beta, int ldc, int deviceID);
+
+void dgemmCov(int transa, int transb, int m, int n,int k, double alpha, long A, int lda,long B,
+              int ldb, double beta, double* C, int ldc, int deviceID);