From 5e7b9e4ec3e2977577f3a41de8ea2cc2203d6973 Mon Sep 17 00:00:00 2001 From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:44:16 -0800 Subject: [PATCH] Add caliper to saxpy (#112) * Add caliper to saxpy * add license * Fix license --------- Co-authored-by: Riyaz Haque Co-authored-by: pearce8 --- repo/saxpy/CMakeLists.txt | 43 +++++++++++++++++++++++++------ repo/saxpy/config.hh.in | 19 ++++++++++++++ repo/saxpy/package.py | 13 +++++----- repo/saxpy/saxpy.cc | 54 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 16 deletions(-) create mode 100644 repo/saxpy/config.hh.in diff --git a/repo/saxpy/CMakeLists.txt b/repo/saxpy/CMakeLists.txt index 8382198cd..fc0c5945b 100644 --- a/repo/saxpy/CMakeLists.txt +++ b/repo/saxpy/CMakeLists.txt @@ -8,25 +8,35 @@ cmake_minimum_required(VERSION 3.23) option(USE_OPNEMP "Use OpenMP" OFF) option(USE_CUDA "Use CUDA" OFF) option(USE_HIP "Use HIP" OFF) +option(USE_CALIPER "Enable Caliper" FALSE) if(USE_CUDA) - project(saxpy VERSION 1.0 LANGUAGES CXX CUDA) + project(saxpy VERSION 1.0.0 LANGUAGES CXX CUDA) else() - project(saxpy VERSION 1.0 LANGUAGES CXX) + project(saxpy VERSION 1.0.0 LANGUAGES CXX) endif() set(SOURCES saxpy.cc) +set(SAXPY_DEPENDENCIES ) + add_executable(${PROJECT_NAME} ${SOURCES}) +find_package(MPI REQUIRED) +list(APPEND SAXPY_DEPENDENCIES + MPI::MPI_CXX) + if(USE_OPENMP) - target_compile_options(saxpy PRIVATE -DUSE_OPENMP -fopenmp) - target_link_options(${PROJECT_NAME} PRIVATE -fopenmp) + target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_OPENMP) + find_package(OpenMP REQUIRED) + list(APPEND SAXPY_DEPENDENCIES + OpenMP::OpenMP_CXX) endif() if(USE_HIP) include_directories(${ROCM_PATH}/include) - target_compile_options(saxpy PRIVATE -x hip -DUSE_HIP --offload-arch=${ROCM_ARCH}) - target_link_libraries(${PROJECT_NAME} PRIVATE -L${ROCM_PATH}/lib -lamdhip64) + target_compile_options(${PROJECT_NAME} PRIVATE -x hip -DUSE_HIP --offload-arch=${ROCM_ARCH}) + list(APPEND SAXPY_DEPENDENCIES + "-L${ROCM_PATH}/lib -lamdhip64") endif() if(USE_CUDA) @@ -34,9 +44,26 @@ if(USE_CUDA) set(CMAKE_CUDA_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CUDA_FLAGS "-Xcompiler -DUSE_CUDA") - set_target_properties(saxpy PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) +endif() + +if (USE_CALIPER) + find_package(caliper REQUIRED) + list(APPEND SAXPY_DEPENDENCIES + caliper) + find_package(adiak REQUIRED) + list(APPEND SAXPY_DEPENDENCIES + adiak::adiak) + target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_CALIPER) endif() -install(TARGETS saxpy +configure_file( + "${CMAKE_SOURCE_DIR}/config.hh.in" + "${CMAKE_CURRENT_SOURCE_DIR}/config.hh" +) + +target_link_libraries(${PROJECT_NAME} PRIVATE ${SAXPY_DEPENDENCIES}) + +install(TARGETS ${PROJECT_NAME} DESTINATION bin ) diff --git a/repo/saxpy/config.hh.in b/repo/saxpy/config.hh.in new file mode 100644 index 000000000..6c0b8b205 --- /dev/null +++ b/repo/saxpy/config.hh.in @@ -0,0 +1,19 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other +// Benchpark Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef SAXPY_CONFIG_HH +#define SAXPY_CONFIG_HH + +#define SAXPY_COMPILER_ID "@CMAKE_C_COMPILER_ID@" +#define SAXPY_COMPILER_VERSION "@CMAKE_C_COMPILER_VERSION@" +#define SAXPY_CMAKE_C_FLAGS "@CMAKE_C_FLAGS@" + +// Version information +#define SAXPY_MAJOR_VERSION @SAXPY_MAJOR_VERSION@ +#define SAXPY_MINOR_VERSION @SAXPY_MINOR_VERSION@ +#define SAXPY_PATCH_VERSION @SAXPY_PATCH_VERSION@ +#define SAXPY_VERSION "@SAXPY_VERSION@" + +#endif // SAXPY_CONFIG_HH diff --git a/repo/saxpy/package.py b/repo/saxpy/package.py index 7e36aa00e..2a4310094 100644 --- a/repo/saxpy/package.py +++ b/repo/saxpy/package.py @@ -22,6 +22,7 @@ class Saxpy(CMakePackage, CudaPackage, ROCmPackage): version('1.0.0') variant("openmp", default=True, description="Enable OpenMP support") + variant("caliper", default=False, description="Enable Caliper monitoring") conflicts("+cuda", when="+rocm+openmp") conflicts("+rocm", when="+cuda+openmp") @@ -29,11 +30,13 @@ class Saxpy(CMakePackage, CudaPackage, ROCmPackage): depends_on("cmake") depends_on("mpi") + depends_on("caliper", when="+caliper") + depends_on("adiak", when="+caliper") @run_before("cmake") def stage_source(self): repo_path = os.path.dirname(spack.repo.PATH.package_path(self.name)) - for f in ["CMakeLists.txt", "saxpy.cc"]: + for f in ["CMakeLists.txt", "saxpy.cc", "config.hh.in"]: shutil.copy2(os.path.join(repo_path, f), self.stage.source_path) def setup_build_environment(self, env): @@ -54,11 +57,7 @@ def cmake_args(self): args.append('-DCMAKE_CXX_COMPILER={0}'.format(spec["mpi"].mpicxx)) - if '+openmp' in spec: - args.append('-DUSE_OPENMP=ON') - else: - args.append("-DUSE_OPENMP=OFF") - + args.append(self.define_from_variant("USE_OPENMP", "openmp")) if '+cuda' in spec: args.append('-DCMAKE_CUDA_HOST_COMPILER={0}'.format(spec["mpi"].mpicxx)) args.append('-DCMAKE_CUDA_COMPILER={0}'.format(spec["cuda"].prefix + "/bin/nvcc")) @@ -80,4 +79,6 @@ def cmake_args(self): rocm_arch = rocm_arch_sorted[0] args.append("-DROCM_ARCH={0}".format(rocm_arch)) + args.append(self.define_from_variant("USE_CALIPER", "caliper")) + return args diff --git a/repo/saxpy/saxpy.cc b/repo/saxpy/saxpy.cc index 74f9f8a03..904ec6a03 100644 --- a/repo/saxpy/saxpy.cc +++ b/repo/saxpy/saxpy.cc @@ -37,6 +37,13 @@ #define GLOBAL #endif +#ifdef USE_CALIPER +#include +#include +#endif + +#include "config.hh" + constexpr int NB = 4; constexpr int NT = 256; @@ -94,6 +101,8 @@ int main(int argc, char** argv) { int opt; int N = 0; + MPI_Comm comm = MPI_COMM_WORLD; + while ((opt = getopt(argc, argv, "n:")) != -1) { switch (opt) { case 'n': @@ -110,8 +119,30 @@ int main(int argc, char** argv) { int rank, size; MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + #ifdef USE_CALIPER + /*----------------------------------------------------------- + * Set Caliper and Adiak metadata + *----------------------------------------------------------*/ + adiak_init(&comm); + adiak_user(); + adiak_uid(); + adiak_launchdate(); + adiak_executable(); + adiak_executablepath(); + adiak_libraries(); + adiak_cmdline(); + adiak_hostname(); + adiak_clustername(); + adiak_namevalue("compiler", adiak_general, NULL, "%s", SAXPY_COMPILER_ID); + adiak_namevalue("compiler version", adiak_general, NULL, "%s", SAXPY_COMPILER_VERSION); + CALI_MARK_BEGIN("main"); + + adiak_namevalue("Problem", adiak_general, NULL, "%s", "standard"); + CALI_MARK_BEGIN("problem"); + #endif DTYPE *h_x, *h_y, *h_r; @@ -119,12 +150,24 @@ int main(int argc, char** argv) { h_y = new DTYPE[N]; h_r = new DTYPE[N]; + #ifdef USE_CALIPER + CALI_MARK_BEGIN("setup"); + #endif for (int i = 0; i < N; ++i) { h_x[i] = i; h_y[i] = i*i; } + #ifdef USE_CALIPER + CALI_MARK_END("setup"); + #endif + #ifdef USE_CALIPER + CALI_MARK_BEGIN("kernel"); + #endif kernel_driver(h_r, h_x, h_y, N); + #ifdef USE_CALIPER + CALI_MARK_END("kernel"); + #endif std::cout << "Kernel done (" << rank << "): " << N << std::endl; @@ -132,6 +175,13 @@ int main(int argc, char** argv) { delete[] h_y; delete[] h_r; + #ifdef USE_CALIPER + CALI_MARK_END("problem"); + + CALI_MARK_END("main"); + adiak_fini(); + #endif + MPI_Finalize(); return 0; }