Add caliper to saxpy (#112)

* Add caliper to saxpy * add license * Fix license --------- Co-authored-by: Riyaz Haque <[email protected]> Co-authored-by: pearce8 <[email protected]>
LLNL · Jan 16, 2024 · 5e7b9e4 · 5e7b9e4
1 parent 2447e35
commit 5e7b9e4
Show file tree

Hide file tree

Showing 4 changed files with 113 additions and 16 deletions.
diff --git a/repo/saxpy/CMakeLists.txt b/repo/saxpy/CMakeLists.txt
@@ -8,35 +8,62 @@ cmake_minimum_required(VERSION 3.23)
 option(USE_OPNEMP "Use OpenMP" OFF)
 option(USE_CUDA "Use CUDA" OFF)
 option(USE_HIP "Use HIP" OFF)
+option(USE_CALIPER "Enable Caliper" FALSE)
 
 if(USE_CUDA)
-  project(saxpy VERSION 1.0 LANGUAGES CXX CUDA)
+  project(saxpy VERSION 1.0.0 LANGUAGES CXX CUDA)
 else()
-  project(saxpy VERSION 1.0 LANGUAGES CXX)
+  project(saxpy VERSION 1.0.0 LANGUAGES CXX)
 endif()
 
 set(SOURCES saxpy.cc)
+set(SAXPY_DEPENDENCIES )
+
 add_executable(${PROJECT_NAME} ${SOURCES})
 
+find_package(MPI REQUIRED)
+list(APPEND SAXPY_DEPENDENCIES
+  MPI::MPI_CXX)
+
 if(USE_OPENMP)
-    target_compile_options(saxpy PRIVATE -DUSE_OPENMP -fopenmp)
-    target_link_options(${PROJECT_NAME} PRIVATE -fopenmp)
+    target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_OPENMP)
+    find_package(OpenMP REQUIRED)
+    list(APPEND SAXPY_DEPENDENCIES
+      OpenMP::OpenMP_CXX)
 endif()
 
 if(USE_HIP)
     include_directories(${ROCM_PATH}/include)
-    target_compile_options(saxpy PRIVATE -x hip -DUSE_HIP --offload-arch=${ROCM_ARCH})
-    target_link_libraries(${PROJECT_NAME} PRIVATE -L${ROCM_PATH}/lib -lamdhip64)
+    target_compile_options(${PROJECT_NAME} PRIVATE -x hip -DUSE_HIP --offload-arch=${ROCM_ARCH})
+    list(APPEND SAXPY_DEPENDENCIES
+      "-L${ROCM_PATH}/lib -lamdhip64")
 endif()
 
 if(USE_CUDA)
     set_source_files_properties(saxpy.cc PROPERTIES LANGUAGE "CUDA")
     set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
     set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
     set(CMAKE_CUDA_FLAGS "-Xcompiler -DUSE_CUDA")
-    set_target_properties(saxpy PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+    set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+endif()
+
+if (USE_CALIPER)
+  find_package(caliper REQUIRED)
+  list(APPEND SAXPY_DEPENDENCIES 
+    caliper)
+  find_package(adiak REQUIRED)
+  list(APPEND SAXPY_DEPENDENCIES
+    adiak::adiak)
+  target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_CALIPER)
 endif()
 
-install(TARGETS saxpy
+configure_file(
+  "${CMAKE_SOURCE_DIR}/config.hh.in"
+  "${CMAKE_CURRENT_SOURCE_DIR}/config.hh"
+)
+
+target_link_libraries(${PROJECT_NAME} PRIVATE ${SAXPY_DEPENDENCIES})
+
+install(TARGETS ${PROJECT_NAME}
         DESTINATION bin
 )
diff --git a/repo/saxpy/config.hh.in b/repo/saxpy/config.hh.in
@@ -0,0 +1,19 @@
+// Copyright 2023 Lawrence Livermore National Security, LLC and other
+// Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef SAXPY_CONFIG_HH
+#define SAXPY_CONFIG_HH
+
+#define SAXPY_COMPILER_ID "@CMAKE_C_COMPILER_ID@"
+#define SAXPY_COMPILER_VERSION "@CMAKE_C_COMPILER_VERSION@"
+#define SAXPY_CMAKE_C_FLAGS "@CMAKE_C_FLAGS@"
+
+// Version information
+#define SAXPY_MAJOR_VERSION @SAXPY_MAJOR_VERSION@
+#define SAXPY_MINOR_VERSION @SAXPY_MINOR_VERSION@
+#define SAXPY_PATCH_VERSION @SAXPY_PATCH_VERSION@
+#define SAXPY_VERSION       "@SAXPY_VERSION@"
+
+#endif // SAXPY_CONFIG_HH
diff --git a/repo/saxpy/package.py b/repo/saxpy/package.py
@@ -22,18 +22,21 @@ class Saxpy(CMakePackage, CudaPackage, ROCmPackage):
     version('1.0.0')
 
     variant("openmp", default=True, description="Enable OpenMP support")
+    variant("caliper", default=False, description="Enable Caliper monitoring")
 
     conflicts("+cuda", when="+rocm+openmp")
     conflicts("+rocm", when="+cuda+openmp")
     conflicts("+openmp", when="+rocm+cuda")
 
     depends_on("cmake")
     depends_on("mpi")
+    depends_on("caliper", when="+caliper")
+    depends_on("adiak", when="+caliper")
 
     @run_before("cmake")
     def stage_source(self):
         repo_path = os.path.dirname(spack.repo.PATH.package_path(self.name))
-        for f in ["CMakeLists.txt", "saxpy.cc"]:
+        for f in ["CMakeLists.txt", "saxpy.cc", "config.hh.in"]:
             shutil.copy2(os.path.join(repo_path, f), self.stage.source_path)
 
     def setup_build_environment(self, env):
@@ -54,11 +57,7 @@ def cmake_args(self):
 
         args.append('-DCMAKE_CXX_COMPILER={0}'.format(spec["mpi"].mpicxx))
 
-        if '+openmp' in spec:
-            args.append('-DUSE_OPENMP=ON')
-        else:
-            args.append("-DUSE_OPENMP=OFF")
-
+        args.append(self.define_from_variant("USE_OPENMP", "openmp"))
         if '+cuda' in spec:
             args.append('-DCMAKE_CUDA_HOST_COMPILER={0}'.format(spec["mpi"].mpicxx))
             args.append('-DCMAKE_CUDA_COMPILER={0}'.format(spec["cuda"].prefix + "/bin/nvcc"))
@@ -80,4 +79,6 @@ def cmake_args(self):
                 rocm_arch = rocm_arch_sorted[0]
                 args.append("-DROCM_ARCH={0}".format(rocm_arch))
 
+        args.append(self.define_from_variant("USE_CALIPER", "caliper"))
+
         return args
diff --git a/repo/saxpy/saxpy.cc b/repo/saxpy/saxpy.cc
@@ -37,6 +37,13 @@
 #define GLOBAL
 #endif
 
+#ifdef USE_CALIPER
+#include <caliper/cali.h>
+#include <adiak.h>
+#endif
+
+#include "config.hh"
+
 constexpr int NB = 4;
 constexpr int NT = 256;
 
@@ -94,6 +101,8 @@ int main(int argc, char** argv) {
     int opt;
     int N = 0;
 
+    MPI_Comm comm = MPI_COMM_WORLD;
+
     while ((opt = getopt(argc, argv, "n:")) != -1) {
         switch (opt) {
             case 'n':
@@ -110,28 +119,69 @@ int main(int argc, char** argv) {
 
     int rank, size;
     MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    MPI_Comm_rank(comm, &rank);
+    MPI_Comm_size(comm, &size);
+
+    #ifdef USE_CALIPER
+      /*-----------------------------------------------------------
+       * Set Caliper and Adiak metadata
+       *----------------------------------------------------------*/
+      adiak_init(&comm);
+      adiak_user();
+      adiak_uid();
+      adiak_launchdate();
+      adiak_executable();
+      adiak_executablepath();
+      adiak_libraries();
+      adiak_cmdline();
+      adiak_hostname();
+      adiak_clustername();
+      adiak_namevalue("compiler", adiak_general, NULL, "%s", SAXPY_COMPILER_ID);
+      adiak_namevalue("compiler version", adiak_general, NULL, "%s", SAXPY_COMPILER_VERSION);
+      CALI_MARK_BEGIN("main");
+
+      adiak_namevalue("Problem", adiak_general, NULL, "%s", "standard");
+      CALI_MARK_BEGIN("problem");
+    #endif
 
     DTYPE *h_x, *h_y, *h_r;
 
     h_x = new DTYPE[N];
     h_y = new DTYPE[N];
     h_r = new DTYPE[N];
 
+    #ifdef USE_CALIPER
+      CALI_MARK_BEGIN("setup");
+    #endif
     for (int i = 0; i < N; ++i) {
       h_x[i] = i;
       h_y[i] = i*i;
     }
+    #ifdef USE_CALIPER
+      CALI_MARK_END("setup");
+    #endif
 
+    #ifdef USE_CALIPER
+      CALI_MARK_BEGIN("kernel");
+    #endif
     kernel_driver(h_r, h_x, h_y, N);
+    #ifdef USE_CALIPER
+      CALI_MARK_END("kernel");
+    #endif
 
     std::cout << "Kernel done (" << rank << "): " << N << std::endl;
 
     delete[] h_x;
     delete[] h_y;
     delete[] h_r;
 
+    #ifdef USE_CALIPER
+      CALI_MARK_END("problem");
+
+      CALI_MARK_END("main");
+      adiak_fini();
+    #endif
+
     MPI_Finalize();
     return 0;
 }