diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index aabaa02be..b58487a45 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -844,3 +844,39 @@ jobs:
benchmark_spec: ad
system_name: tioga
system_spec: llnl-elcapitan rocm=6.2.4 compiler=rocmcc
+
+ - name: branson/openmp caliper=mpi,time ruby llnl-cluster cluster=ruby compiler=intel
+ uses: ./.github/actions/dynamic-dry-run
+ with:
+ benchmark_name: branson
+ benchmark_mode: openmp
+ benchmark_spec: branson+openmp caliper=mpi,time
+ system_name: ruby
+ system_spec: llnl-cluster cluster=ruby compiler=intel
+
+ - name: branson/cuda caliper=cuda,time lassen llnl-sierra cuda=11-8-0 compiler=clang-ibm
+ uses: ./.github/actions/dynamic-dry-run
+ with:
+ benchmark_name: branson
+ benchmark_mode: cuda
+ benchmark_spec: branson+cuda caliper=cuda,time
+ system_name: lassen
+ system_spec: llnl-sierra cuda=11-8-0 compiler=clang-ibm
+
+ - name: branson/cuda caliper=cuda,time venado lanl-venado cuda=12.5 compiler=cce +gtl
+ uses: ./.github/actions/dynamic-dry-run
+ with:
+ benchmark_name: branson
+ benchmark_mode: cuda
+ benchmark_spec: branson+cuda caliper=cuda,time
+ system_name: venado
+ system_spec: lanl-venado cuda=12.5 compiler=cce +gtl
+
+ - name: branson/rocm caliper=mpi,time tioga llnl-elcapitan rocm=6.2.4 compiler=cce +gtl
+ uses: ./.github/actions/dynamic-dry-run
+ with:
+ benchmark_name: branson
+ benchmark_mode: rocm
+ benchmark_spec: branson+rocm caliper=mpi,time
+ system_name: tioga
+ system_spec: llnl-elcapitan rocm=6.2.4 compiler=cce +gtl
diff --git a/experiments/branson/experiment.py b/experiments/branson/experiment.py
new file mode 100644
index 000000000..fa334dbd3
--- /dev/null
+++ b/experiments/branson/experiment.py
@@ -0,0 +1,125 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.error import BenchparkError
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.openmp import OpenMPExperiment
+from benchpark.cuda import CudaExperiment
+from benchpark.rocm import ROCmExperiment
+from benchpark.scaling import StrongScaling
+from benchpark.scaling import WeakScaling
+from benchpark.caliper import Caliper
+
+
+class Branson(
+ Experiment,
+ OpenMPExperiment,
+ CudaExperiment,
+ ROCmExperiment,
+ StrongScaling,
+ WeakScaling,
+ Caliper,
+):
+ variant(
+ "workload",
+ default="branson",
+ description="workload name",
+ )
+
+ variant(
+ "version",
+ default="develop",
+ description="app version",
+ )
+
+ variant(
+ "n_groups",
+ default="30",
+ values=int,
+ description="Number of groups",
+ )
+
+ def compute_applications_section(self):
+ # TODO: Replace with conflicts clause
+ scaling_modes = {
+ "strong": self.spec.satisfies("+strong"),
+ "weak": self.spec.satisfies("+weak"),
+ "single_node": self.spec.satisfies("+single_node"),
+ }
+
+ scaling_mode_enabled = [key for key, value in scaling_modes.items() if value]
+ if len(scaling_mode_enabled) != 1:
+ raise BenchparkError(
+ f"Only one type of scaling per experiment is allowed for application package {self.name}"
+ )
+
+ # Number of processes in each dimension
+ num_nodes = {"n_nodes": 1}
+
+ # Per-process size (in zones) in each dimension
+ num_particles = {"num_particles": 850000000}
+
+ if self.spec.satisfies("+single_node"):
+ for pk, pv in num_nodes.items():
+ self.add_experiment_variable(pk, pv, True)
+ for nk, nv in num_particles.items():
+ self.add_experiment_variable(nk, nv, True)
+ elif self.spec.satisfies("+strong"):
+ scaled_variables = self.generate_strong_scaling_params(
+ {tuple(num_nodes.keys()): list(num_nodes.values())},
+ int(self.spec.variants["scaling-factor"][0]),
+ int(self.spec.variants["scaling-iterations"][0]),
+ )
+ for pk, pv in scaled_variables.items():
+ self.add_experiment_variable(pk, pv, True)
+ for nk, nv in num_particles.items():
+ self.add_experiment_variable(nk, nv, True)
+ elif self.spec.satisfies("+weak"):
+ scaled_variables = self.generate_weak_scaling_params(
+ {tuple(num_nodes.keys()): list(num_nodes.values())},
+ {tuple(num_particles.keys()): list(num_particles.values())},
+ int(self.spec.variants["scaling-factor"][0]),
+ int(self.spec.variants["scaling-iterations"][0]),
+ )
+ for k, v in scaled_variables.items():
+ self.add_experiment_variable(k, v, True)
+
+ self.add_experiment_variable(
+ "use_gpu",
+ (
+ "TRUE"
+ if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm")
+ else "FALSE"
+ ),
+ )
+
+ self.add_experiment_variable("n_ranks", "{n_nodes}*{sys_cores_per_node}", True)
+
+ def compute_spack_section(self):
+ # get package version
+ app_version = self.spec.variants["version"][0]
+
+ # get system config options
+ # TODO: Get compiler/mpi/package handles directly from system.py
+ system_specs = {}
+ system_specs["compiler"] = "default-compiler"
+ system_specs["mpi"] = "default-mpi"
+ if self.spec.satisfies("+cuda"):
+ system_specs["cuda_version"] = "{default_cuda_version}"
+ system_specs["cuda_arch"] = "{cuda_arch}"
+ if self.spec.satisfies("+rocm"):
+ system_specs["rocm_arch"] = "{rocm_arch}"
+
+ # set package spack specs
+ self.add_spack_spec(system_specs["mpi"])
+
+ self.add_spack_spec(
+ self.name,
+ [
+ f"branson@{app_version} +metis n_groups={self.spec.variants['n_groups'][0]} ",
+ system_specs["compiler"],
+ ],
+ )
diff --git a/legacy/experiments/branson/mpi-only/ramble.yaml b/legacy/experiments/branson/mpi-only/ramble.yaml
new file mode 100644
index 000000000..c4cb4e929
--- /dev/null
+++ b/legacy/experiments/branson/mpi-only/ramble.yaml
@@ -0,0 +1,51 @@
+ramble:
+ applications:
+ branson:
+ workloads:
+ branson:
+ experiments:
+ branson_branson_weak_scaling_caliper_time_mpi_{n_nodes}_{num_particles}_{n_ranks}:
+ exclude: {}
+ matrix: []
+ variables:
+ n_nodes:
+ - 1
+ - 2
+ - 4
+ - 8
+ n_ranks: '{n_nodes}*{sys_cores_per_node}'
+ num_particles:
+ - 850000000
+ - 1700000000
+ - 3400000000
+ - 6800000000
+ variants:
+ package_manager: spack
+ zips: {}
+ config:
+ deprecated: true
+ spack_flags:
+ concretize: -U -f
+ install: --add --keep-stage
+ include:
+ - ./configs
+ modifiers:
+ - name: allocation
+ - mode: mpi
+ name: caliper
+ - mode: time
+ name: caliper
+ software:
+ environments:
+ branson:
+ packages:
+ - caliper
+ - default-mpi
+ - branson
+ packages:
+ branson:
+ compiler: default-compiler
+ pkg_spec: branson@develop+caliper
+ caliper:
+ compiler: default-compiler
+ pkg_spec: caliper@master+adiak+mpi~libunwind~libdw~papi
diff --git a/legacy/experiments/branson/openmp/ramble.yaml b/legacy/experiments/branson/openmp/ramble.yaml
deleted file mode 100644
index e90dc280a..000000000
--- a/legacy/experiments/branson/openmp/ramble.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
-# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: Apache-2.0
-ramble:
- include:
- - ./configs/software.yaml
- - ./configs/variables.yaml
- - ./configs/modifier.yaml
- config:
- deprecated: true
- spack_flags:
- install: '--add --keep-stage'
- concretize: '-U -f'
-#Branson does not currently work on lassen
- modifiers:
- - name: allocation
- applications:
- branson:
- workloads:
- branson:
- variables:
- n_ranks: '{n_nodes}*{sys_cores_per_node}'
- experiments:
- branson_strong_{n_nodes}:
- variants:
- package_manager: spack
- variables:
- #looks like branson should use ~25% of memory per node
- num_particles: '850000000'
- n_nodes: ['1','2','4','8']
-
- branson_weak_{n_nodes}:
- variants:
- package_manager: spack
- variables:
- num_particles: '850000000*{n_nodes}'
- n_nodes: ['1','2','4','8']
- software:
- packages:
- branson:
- pkg_spec: branson@develop
- compiler: default-compiler
- environments:
- branson:
- packages:
- - default-mpi
- - branson
- - '{modifier_package_name}'
diff --git a/repo/branson/application.py b/repo/branson/application.py
index 2103c1ca5..51a5e246a 100644
--- a/repo/branson/application.py
+++ b/repo/branson/application.py
@@ -19,14 +19,15 @@ class Branson(ExecutableApplication):
executable('setup_experiment',
template=[
'cp {branson}/inputs/* {experiment_run_dir}/.',
- 'sed -i "s|250000000|{num_particles}|g" {experiment_run_dir}/{input_file}'
+ 'sed -i "s|[0-9]*|{num_particles}|g" {experiment_run_dir}/{input_file}',
+ 'sed -i "s|.*|{use_gpu}|g" {experiment_run_dir}/{input_file}'
])
executable('p', '{branson}/bin/BRANSON {experiment_run_dir}/{input_file}', use_mpi=True)
workload('branson', executables=['setup_experiment','p'])
- workload_variable('input_file', default='3D_hohlraum_multi_node.xml',
+ workload_variable('input_file', default='3D_hohlraum_single_node.xml',
description='input file name',
workloads=['branson'])
diff --git a/repo/branson/branson_cmake.patch b/repo/branson/branson_cmake.patch
new file mode 100644
index 000000000..7e1891cf7
--- /dev/null
+++ b/repo/branson/branson_cmake.patch
@@ -0,0 +1,531 @@
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -9,65 +9,184 @@
+ # Enbable debug mode by passing -DCMAKE_BUILD_TYPE=Debug to CMake, default is
+ # Release
+
+-cmake_minimum_required (VERSION 3.11)
++cmake_minimum_required(VERSION 3.23)
+
+-option(USE_GPU "user-set flag to compile in GPU code" FALSE)
++project (BRANSON
++ VERSION 0.8
++ DESCRIPTION "Branson can be used to study different algorithms for parallel Monte Carlo transport. Currently it contains particle passing and mesh passing methods for domain decomposition."
++)
+
+-if(DEFINED ENV{CUDADIR}
+- OR DEFINED ENV{CUDACXX}
+- OR DEFINED ENV{CUDA_HOME})
+- if(USE_GPU)
+- set(GPU_DBS_STRING "CUDA" CACHE STRING "If CUDA is available, this variable is 'CUDA'")
+- enable_language(CUDA)
+- endif()
+-elseif(
+- "$ENV{LOADEDMODULES}" MATCHES "rocmcc"
+- OR DEFINED ENV{HIPCXX}
+- OR DEFINED CMAKE_HIP_COMPILER
+- OR DEFINED ENV{ROCM_PATH})
+- if(USE_GPU)
+- set(GPU_DBS_STRING "HIP" CACHE STRING "If HIP is available, this variable is 'HIP'")
+- #if(CMAKE_HIP_COMPILER MATCHES "CC")
+- # set(CMAKE_HIP_FLAGS "-x hip")
+- #endif()
+- enable_language(HIP)
++## Fail if someone tries to config an in-source build.
++if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
++ message(FATAL_ERROR "In-source builds are not supported. Please remove "
++ "CMakeCache.txt from the 'src' dir and configure an "
++ "out-of-source build in another directory.")
++endif()
++
++# Set the build type to Release by default if not set
++if(NOT CMAKE_BUILD_TYPE)
++ set(CMAKE_BUILD_TYPE "Release")
++endif()
++
++set(CMAKE_CXX_STANDARD 17)
++set(CMAKE_CXX_STANDARD_REQUIRED ON)
++set(CMAKE_CXX_EXTENSIONS OFF)
++string(REPLACE " " ";" CMAKE_BRANSON_CXXFLAGS "${CMAKE_CXX_FLAGS}")
++
++option(ENABLE_CUDA "Use CUDA" FALSE)
++option(ENABLE_HIP "Use HIP" FALSE)
++option(ENABLE_CALIPER "Enable Caliper" FALSE)
++option(ENABLE_OPENMP "Enable OpenMP" FALSE)
++option(ENABLE_METIS "Enable METIS" FALSE)
++option(ENABLE_VIZ "Enable VIZ" FALSE)
++
++set(CMAKE_VERBOSE_MAKEFILE ON)
++
++find_package(MPI REQUIRED)
++list(APPEND branson_deps
++ MPI::MPI_CXX)
++
++##############################################################################
++# OpenMP
++##############################################################################
++if(ENABLE_OPENMP)
++ message(STATUS "Looking for OpenMP...")
++ find_package(OpenMP REQUIRED)
++ if(OpenMP_FOUND)
++ message(STATUS "Looking for openmp......found ${OpenMP_CXX_FLAGS}")
++ set(USE_OPENMP ON)
++ list(APPEND branson_deps
++ OpenMP::OpenMP_CXX
++ )
+ endif()
++else()
++ message(STATUS "Disabling OpenMP...")
++ set(USE_OPENMP OFF)
+ endif()
+
+-message("GPU_DBS_STRING: ${GPU_DBS_STRING}")
+-project (BRANSON
+- VERSION 0.8
+- DESCRIPTION "Branson can be used to study different algorithms for parallel Monte Carlo transport. Currently it contains particle passing and mesh passing methods for domain decomposition."
+-# HOMEPAGE URL "https://github.com/lanl/branson" # needs cmake 3.12+
+- LANGUAGES CXX C ${GPU_DBS_STRING})
+-
+-get_property(_LANGUAGES_ GLOBAL PROPERTY ENABLED_LANGUAGES)
+-
+-message("Languages: ${_LANGUAGES_}")
+-if((_LANGUAGES_ MATCHES CUDA OR _LANGUAGES_ MATCHES HIP) AND USE_GPU)
+- message("CUDA/HIP module found (CUDA/HIP environment variables set) and USE_GPU is on, making GPU BRANSON")
+- add_compile_definitions(HAS_GPU)
+-elseif(USE_GPU)
+- message(FATAL_ERROR "CUDA/HIP module NOT found (CUDA/HIP environment variables set) but USE_GPU is on, reconfigure with USE_GPU off or fix modules")
+-elseif(NOT USE_GPU)
+- message("GPU mode not requested, making CPU only BRANSON")
++if(ENABLE_CUDA)
++ enable_language(CXX)
++
++ find_package(CUDAToolkit REQUIRED)
++ list(APPEND branson_deps
++ CUDA::cudart)
++
++ include(CheckLanguage)
++ check_language(CUDA)
++
++ set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
++
++ if(NOT CMAKE_CUDA_COMPILER)
++ message(FATAL_ERROR "Unable to find the nvcc compiler. Please use"
++ "CMAKE_CUDA_COMPILER to provide the nvcc compiler.")
++ endif()
++
++ enable_language(CUDA)
++ set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
++ set(CMAKE_CUDA_STANDARD_REQUIRED ON)
++ set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}")
++ set_source_files_properties("main.cc" PROPERTIES LANGUAGE CUDA)
++ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xptxas -v")
++ set(CMAKE_CUDA_FLAGS "-g --expt-relaxed-constexpr ${CMAKE_CUDA_FLAGS}")
++ set(CMAKE_CUDA_FLAGS "-expt-extended-lambda ${CMAKE_CUDA_FLAGS}")
++ list(APPEND CMAKE_BRANSON_CXXFLAGS -DUSE_GPU)
++ list(APPEND CMAKE_BRANSON_CXXFLAGS -DHAS_GPU)
++
++ set(GPU_DBS_STRING "CUDA" CACHE STRING "If CUDA is available, this variable is 'CUDA'")
++
++ message("Making GPU(CUDA) BRANSON")
++
++elseif(ENABLE_HIP)
++ if(NOT DEFINED HIP_PATH)
++ if(NOT DEFINED ENV{HIP_PATH})
++ set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed")
++ else()
++ set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed")
++ endif()
++ endif()
++
++ if(NOT DEFINED ROCM_PATH)
++ if(DEFINED ENV{ROCM_PATH})
++ set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed")
++ elseif(DEFINED ENV{HIP_PATH})
++ set(ROCM_PATH "$ENV{HIP_PATH}/.." CACHE PATH "Path to which ROCM has been installed")
++ else()
++ set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCM has been installed")
++ endif()
++ endif()
++
++ if(NOT DEFINED HCC_PATH)
++ if(DEFINED ENV{HCC_PATH})
++ set(HCC_PATH $ENV{HCC_PATH} CACHE PATH "Path to which HCC has been installed")
++ else()
++ set(HCC_PATH "${ROCM_PATH}/hcc" CACHE PATH "Path to which HCC has been installed")
++ endif()
++ set(HCC_HOME "${HCC_PATH}")
++ endif()
++
++ if(NOT DEFINED HIP_CLANG_PATH)
++ if(NOT DEFINED ENV{HIP_CLANG_PATH})
++ set(HIP_CLANG_PATH "${ROCM_PATH}/llvm/bin" CACHE PATH "Path to which HIP compatible clang binaries have been installed")
++ else()
++ set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH} CACHE PATH "Path to which HIP compatible clang binaries have been installed")
++ endif()
++ endif()
++
++ set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
++ list(APPEND CMAKE_PREFIX_PATH
++ "${HIP_PATH}/lib/cmake"
++ "${HIP_PATH}/../lib/cmake" # hopefully catches all extra HIP dependencies
++ )
++
++ find_package(HIP QUIET)
++ if(HIP_FOUND)
++ message(STATUS "Found HIP: " ${HIP_VERSION})
++ else()
++ message(FATAL_ERROR "Could not find HIP. Ensure that HIP is either installed in /opt/rocm/hip or the variable HIP_PATH is set to point to the right location.")
++ endif()
++ find_package(hip REQUIRED)
++
++ # For ROCm >=3.5, wipe hip-clang specific interface options which are propagated
++ set_target_properties(hip::device PROPERTIES INTERFACE_COMPILE_OPTIONS "-fPIC")
++ set_target_properties(hip::device PROPERTIES INTERFACE_LINK_LIBRARIES "hip::host")
++
++ list(APPEND branson_deps
++ "-fgpu-rdc"
++ "--hip-link"
++ "--offload-arch=${HIP_ARCH}"
++ "--rocm-path=${ROCM_PATH}"
++ hip::host
++ hip::device)
++ list(APPEND CMAKE_BRANSON_CXXFLAGS -DUSE_GPU)
++ list(APPEND CMAKE_BRANSON_CXXFLAGS -DHAS_GPU)
++ list(APPEND CMAKE_BRANSON_CXXFLAGS -x hip)
++ list(APPEND CMAKE_BRANSON_CXXFLAGS "--offload-arch=${HIP_ARCH}")
++ list(APPEND CMAKE_BRANSON_CXXFLAGS "-fgpu-rdc")
++ list(APPEND CMAKE_BRANSON_CXXFLAGS "-Wno-unused-result")
++
++ #set_source_files_properties("main.cc" PROPERTIES LANGUAGE HIP)
++
++ set(GPU_DBS_STRING "HIP" CACHE STRING "If HIP is available, this variable is 'HIP'")
++
++ message("Making GPU(HIP) BRANSON")
++
+ else()
+- message("CUDA/HIP module not found and GPU mode not requested, making CPU only BRANSON")
++ message("Making CPU BRANSON")
++
+ endif()
+
++message("GPU_DBS_STRING: ${GPU_DBS_STRING}")
+
+ # Build system support files are located here.
+-set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/config )
++set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/config ${CMAKE_MODULE_PATH})
+
+-message( "
+-This is ${CMAKE_PROJECT_NAME} v. ${PROJECT_VERSION}.
+-")
++message("This is ${CMAKE_PROJECT_NAME} v. ${PROJECT_VERSION}.")
+
+ #------------------------------------------------------------------------------#
+ # User configurable options
+
+ set (BRANSON_N_GROUPS ${N_GROUPS})
+-if (NOT BRANSON_N_GROUPS)
++if(NOT BRANSON_N_GROUPS)
+ message(STATUS
+ "HARDCODED NUMBER OF GROUPS NOT SET, DEFAULTING TO ONE GROUP (GRAY)
+ use '-DN_GROUPS=' when running cmake to select multigroup.\n")
+@@ -83,28 +202,10 @@ set(ENABLE_VERBOSE_GPU_TRANSPORT ${ENABLE_VERBOSE_GPU_TRANSPORT} CACHE STRING "F
+ #------------------------------------------------------------------------------#
+ # Record the site name
+
+-site_name( SITENAME )
+-string( REGEX REPLACE "([A-z0-9]+).*" "\\1" SITENAME ${SITENAME} )
++site_name(SITENAME)
++string(REGEX REPLACE "([A-z0-9]+).*" "\\1" SITENAME ${SITENAME})
+ include (lanl-setup)
+-set( SITENAME ${SITENAME} CACHE "STRING" "Name of the current machine" FORCE)
+-
+-#------------------------------------------------------------------------------#
+-# Setup compiler options
+-set( CXX_STANDARD_REQUIRED ON )
+-set( CMAKE_CXX_STANDARD 17 )
+-
+-# Do not enable extensions (e.g.: --std=gnu++11)
+-set( CMAKE_CXX_EXTENSIONS OFF )
+-set( CMAKE_C_EXTENSIONS OFF )
+-
+-#------------------------------------------------------------------------------#
+-# Build type and custom compiler flags
+-
+-if ( "${CMAKE_BUILD_TYPE}notset" STREQUAL "notset" AND
+- NOT DEFINED CMAKE_CONFIGURATION_TYPES )
+- set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Release|Debug|RelWithDebInfo"
+- FORCE)
+-endif ()
++set(SITENAME ${SITENAME} CACHE "STRING" "Name of the current machine" FORCE)
+
+ # GCC options for address or undefined sanitizing
+ #set(GCC_SANITIZE_COMPILE_FLAGS "-fsanitize=address")
+@@ -113,135 +214,137 @@ endif ()
+ #set(GCC_SANITIZE_LINK_FLAGS "-fsanitize=undefined")
+
+ # add compiler flags
+-if( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" )
+- set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -ffp-contract=off")
+- set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0")
+- #set( CMAKE_CXX_FLAGS_RELEASE "-march=native -O3 -funroll-loops -fno-var-tracking-assignments")
+- set( CMAKE_CXX_FLAGS_RELEASE "-g -O3 -funroll-loops -fno-var-tracking-assignments")
++if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -ffp-contract=off")
++ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0")
++ #set(CMAKE_CXX_FLAGS_RELEASE "-march=native -O3 -funroll-loops -fno-var-tracking-assignments")
++ set(CMAKE_CXX_FLAGS_RELEASE "-g -O3 -funroll-loops -fno-var-tracking-assignments")
+ # optionally add sanitize flags
+- #set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${GCC_SANITIZE_COMPILE_FLAGS}")
+- #set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_SANITIZE_LINK_FLAGS}")
+-elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Intel" )
+- set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0")
++ #set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${GCC_SANITIZE_COMPILE_FLAGS}")
++ #set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_SANITIZE_LINK_FLAGS}")
++elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
++ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0")
+ # note that the "fp-model=precise" flag significantly limite vectorization and slows down
+ # branson but it makes it the hardware counter data match Jayenne more closely (Jayenne uses
+ # this flag)
+- set( CMAKE_CXX_FLAGS_RELEASE "-g -O3 -fp-speculation=fast -xhost -qno-opt-dynamic-align")
+-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
++ set(CMAKE_CXX_FLAGS_RELEASE "-g -O3 -fp-speculation=fast -xhost -qno-opt-dynamic-align")
++elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -Wall -O0 -xhost -qopenmp")
+ set(CMAKE_CXX_FLAGS_RELEASE "-g -O3 -xhost -qopenmp -qopt-zmm-usage=high -fp-speculation=fast -qopt-report=3 -qopt-report-file=runAVX512.optrpt")
+-elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" )
+- set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DR123_USE_GNU_UINT128=0")
+- set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DR123_USE_GNU_UINT128=0")
++elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Cray")
++ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DR123_USE_GNU_UINT128=0")
++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DR123_USE_GNU_UINT128=0")
+ endif()
+
++if(ENABLE_CUDA)
++ string(JOIN " " CMAKE_BRANSON_CUDAFLAGS ${CMAKE_BRANSON_CXXFLAGS})
++ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler ${CMAKE_BRANSON_CUDAFLAGS}")
++else()
++ add_compile_options(${CMAKE_BRANSON_CXXFLAGS})
++endif()
+
+ #------------------------------------------------------------------------------#
+ # Look for Third Party Libraries (metis, etc.)
+ add_subdirectory(pugixml)
+
+-include(find_tpls)
+-setupTPLs()
+-
+ #------------------------------------------------------------------------------#
+ # Report build configuration
+
+ # Summary of TPLs
+ include(FeatureSummary)
+ message(" ")
+-feature_summary( WHAT ALL
++feature_summary(WHAT ALL
+ INCLUDE_QUIET_PACKAGES
+ FATAL_ON_MISSING_REQUIRED_PACKAGES
+- QUIET_ON_EMPTY )
++ QUIET_ON_EMPTY)
+
+ # Summary of user-selectable build options
+-message( "\nBuild Summary:\n")
+-message( STATUS "Machine name : ${SITENAME}")
++message("\nBuild Summary:\n")
++message(STATUS "Machine name : ${SITENAME}")
+ report_lanl_hpc_features()
+-if( ${BRANSON_N_GROUPS} GREATER 1 )
+- message( STATUS "Energy groups : ${BRANSON_N_GROUPS}")
++if(${BRANSON_N_GROUPS} GREATER 1)
++ message(STATUS "Energy groups : ${BRANSON_N_GROUPS}")
+ else()
+- message( STATUS "Energy groups : Gray (1-group)")
++ message(STATUS "Energy groups : Gray (1-group)")
+ endif()
+-message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+-message( STATUS "Compiler : ${CMAKE_CXX_COMPILER}")
+-if( CMAKE_CONFIGURATION_TYPES )
+- message( STATUS "Compiler Flags (All) : ${CMAKE_CXX_FLAGS}")
+- message( STATUS "Compiler Flags (Debug) : ${CMAKE_CXX_FLAGS_DEBUG}")
+- message( STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}")
++message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
++message(STATUS "Compiler : ${CMAKE_CXX_COMPILER}")
++string(REPLACE ";" " " CMAKE_BRANSON_CXXFLAGS_STR "${CMAKE_BRANSON_CXXFLAGS}")
++if(CMAKE_CONFIGURATION_TYPES)
++ message(STATUS "Compiler Flags (All) : ${CMAKE_CXX_FLAGS} ${CMAKE_BRANSON_CXXFLAGS_STR}")
++ message(STATUS "Compiler Flags (Debug) : ${CMAKE_CXX_FLAGS_DEBUG}")
++ message(STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}")
+ else()
+- string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER )
+- message( STATUS "Compiler Flags : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
++ string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
++ message(STATUS "Compiler Flags : ${CMAKE_CXX_FLAGS} ${CMAKE_BRANSON_CXXFLAGS_STR} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
+ endif()
+ message("\n")
+
+-#------------------------------------------------------------------------------#
+-# Set up libraries for Cray RMA routines
+-
+-# set(DMAPP_DYNAMIC -Wl,--whole-archive,-ldmapp,--no-whole-archive)
+-
+-
+-#------------------------------------------------------------------------------#
+-# Targets
+-file(GLOB headers *.h)
+-add_executable(BRANSON main.cc ${headers})
+-target_include_directories( BRANSON PRIVATE
+- $ ${PROJECT_SOURCE_DIR}/pugixml/src/)
+-set( branson_deps
+- MPI::MPI_CXX
+- MPI::MPI_C
+- pugixml)
+-
+-if(OpenMP_FOUND)
+- set(branson_deps "OpenMP::OpenMP_CXX;${branson_deps}")
+-endif()
+-
+-if(METIS_FOUND)
+- set( branson_deps "METIS::metis;${branson_deps}")
+-endif()
+-
+-if( VIZ_LIBRARIES_FOUND )
+- set( branson_deps "Silo::silo;${HDF5_LIBRARIES};${branson_deps}" )
+-endif()
++list(APPEND branson_deps pugixml)
++
++##############################################################################
++# caliper/adiak
++##############################################################################
++if(ENABLE_CALIPER)
++ message(STATUS "Looking for caliper...")
++ find_package(caliper REQUIRED)
++ if(caliper_FOUND)
++ message(STATUS "Looking for caliper......found ${caliper_DIR}")
++ list(APPEND branson_deps
++ caliper)
++ endif()
+
+-if(caliper_FOUND)
+- set( branson_deps "caliper;${branson_deps}")
++ message(STATUS "Looking for adiak...")
++ find_package(adiak REQUIRED)
++ if(adiak_FOUND)
++ message(STATUS "Looking for adiak......found ${adiak_DIR}")
++ list(APPEND branson_deps
++ adiak::adiak)
++ endif()
++else()
++ message(STATUS "Disabling caliper...")
+ endif()
+
+-#------------------------------------------------------------------------------#
+-# Generate config.h
+-
+-configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h)
+-#------------------------------------------------------------------------------#
+-
+-if("${GPU_DBS_STRING}" STREQUAL "CUDA" )
+- message("Setting CUDA compiler options")
+- #set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "70") # V100
+- set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "80") # A100
+- set_target_properties(BRANSON PROPERTIES CUDA_STANDARD 17)
+- string(APPEND CMAKE_CUDA_FLAGS " -g --expt-relaxed-constexpr")
+- string(APPEND CMAKE_CUDA_FLAGS " --expt-extended-lambda" )
+- set_source_files_properties("main.cc" PROPERTIES LANGUAGE CUDA)
+-elseif("${GPU_DBS_STRING}" STREQUAL "HIP" )
+- message("Setting HIP compiler options")
+- if(CMAKE_HIP_COMPILER MATCHES "CC")
+- set(CMAKE_HIP_FLAGS "-x hip")
++##############################################################################
++# metis
++##############################################################################
++if(ENABLE_METIS)
++ message(STATUS "Looking for METIS...")
++ find_package(METIS REQUIRED)
++ if(METIS_FOUND)
++ message(STATUS "Looking for METIS......found ${METIS_LIBRARIES}")
++ list(APPEND branson_deps
++ METIS::metis)
+ endif()
+- set_target_properties(BRANSON PROPERTIES HIP_ARCHITECTURES "gfx942") # MI300
+- set_target_properties(BRANSON PROPERTIES HIP_STANDARD 17)
+- string(APPEND CMAKE_HIP_FLAGS " -g -mllvm=--disable-peephole")
+- #string(APPEND CMAKE_HIP_FLAGS " --expt-extended-lambda" )
+- set_source_files_properties("main.cc" PROPERTIES LANGUAGE HIP)
+ else()
+- message("GPU Options: Not a GPU build or GPU_DBS_STRING not recognized")
++ message(STATUS "Disabling METIS...")
+ endif()
+
+-target_link_libraries( BRANSON PRIVATE ${branson_deps} )
+-
+-#------------------------------------------------------------------------------#
+-# Testing
++##############################################################################
++# Silo and HDF5 libraries
++##############################################################################
++if(ENABLE_VIZ)
++ message(STATUS "Looking for HDF5...")
++ find_package(HDF5 REQUIRED)
++ if(HDF5_FOUND)
++ list(GET HDF5_LIBRARIES 0 hdf5lib)
++ message(STATUS "Looking for HDF5..found ${hdf5lib}")
++ unset(hdf5lib)
++ message(STATUS "Looking for Silo...")
++ find_package(Silo REQUIRED)
++ if(Silo_FOUND)
++ message(STATUS "Looking for Silo..found ${Silo_LIBRARY}")
++ endif()
++ list(APPEND branson_deps
++ Silo::silo
++ ${HDF5_LIBRARIES}
++ )
++ endif()
++else ()
++ message(STATUS "Disabling visualization libraries...")
++endif()
+
+-option( BUILD_TESTING "Should we compile the tests?" ON )
++option(BUILD_TESTING "Should we compile the tests?" ON)
+ if(BUILD_TESTING)
+ enable_testing()
+ add_subdirectory(test)
+@@ -252,11 +355,38 @@ else()
+ "Building tests disabled, set BUILD_TESTING=TRUE or don't set BUILD_TESTING to enable test builds")
+ endif()
+
++#------------------------------------------------------------------------------#
++# Generate config.h
++
++configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h)
++#------------------------------------------------------------------------------#
++
++#------------------------------------------------------------------------------#
++# Targets
++file(GLOB headers *.h)
++add_executable(BRANSON main.cc ${headers})
++target_link_libraries(BRANSON PUBLIC ${branson_deps})
++target_include_directories(BRANSON PRIVATE
++ ${CMAKE_BINARY_DIR} ${PROJECT_SOURCE_DIR}/pugixml/src/ ${HIP_INCLUDE_DIRS})
++
++if(ENABLE_CUDA)
++ set_target_properties(BRANSON
++ PROPERTIES
++ CUDA_SEPARABLE_COMPILATION ON
++ CUDA_RESOLVE_DEVICE_SYMBOLS ON)
++endif()
++
++if(ENABLE_HIP)
++ set_target_properties(BRANSON
++ PROPERTIES
++ HIP_SEPARABLE_COMPILATION ON
++ HIP_RESOLVE_DEVICE_SYMBOLS ON)
++endif()
+
+ #------------------------------------------------------------------------------#
+ # Targets for installation
+
+-install(TARGETS BRANSON DESTINATION bin)
++install(TARGETS BRANSON DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+ #------------------------------------------------------------------------------#
+ # End src/CMakeLists.txt
diff --git a/repo/branson/branson_power9.patch b/repo/branson/branson_power9.patch
new file mode 100644
index 000000000..ea3110173
--- /dev/null
+++ b/repo/branson/branson_power9.patch
@@ -0,0 +1,11 @@
+--- a/src/random123/features/gccfeatures.h
++++ b/src/random123/features/gccfeatures.h
+@@ -45,7 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ #endif
+
+ #ifdef __powerpc__
+-#include
++//#include
+ #endif
+
+ #ifndef R123_STATIC_INLINE
diff --git a/repo/branson/package.py b/repo/branson/package.py
index 14e31c343..dc31a4b94 100644
--- a/repo/branson/package.py
+++ b/repo/branson/package.py
@@ -6,8 +6,10 @@
from spack.package import *
from spack.pkg.builtin.boost import Boost
+import os
-class Branson(CMakePackage):
+
+class Branson(CMakePackage, CudaPackage, ROCmPackage):
"""Branson's purpose is to study different algorithms for parallel Monte
Carlo transport. Currently it contains particle passing and mesh passing
methods for domain decomposition."""
@@ -30,6 +32,13 @@ class Branson(CMakePackage):
)
version("0.81", sha256="493f720904791f06b49ff48c17a681532c6a4d9fa59636522cf3f9700e77efe4")
version("0.8", sha256="85ffee110f89be00c37798700508b66b0d15de1d98c54328b6d02a9eb2cf1cb8")
+
+ variant("openmp", default=False, description="Enable OpenMP support")
+ variant("caliper", default=False, description="Enable Caliper monitoring")
+ variant("metis", default=False, description="Enable METIS")
+ variant("viz", default=False, description="Enable VIZ")
+ variant("n_groups", default=30, values=int, description="Number of groups")
+
#depends_on("mpi")
depends_on("mpi@2:")
@@ -39,16 +48,73 @@ class Branson(CMakePackage):
depends_on(Boost.with_default_variants, when="@:0.81")
depends_on("metis")
depends_on("parmetis", when="@:0.81")
+ depends_on("caliper", when="+caliper")
+ depends_on("adiak", when="+caliper")
root_cmakelists_dir = "src"
+ flag_handler = build_system_flags
+
+ patch("branson_cmake.patch")
+ patch("branson_power9.patch")
+
+ def setup_build_environment(self, env):
+ if "+cuda" in self.spec:
+ env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler")
+
+ def patch(self):
+ ppu_intrinsics_file = os.path.join(self.stage.source_path, "src", "random123", "features", "ppu_intrinsics.h")
+ with open(ppu_intrinsics_file , "w") as f:
+ pass
+
def cmake_args(self):
spec = self.spec
args = []
- #args.append("--enable-mpi")
- args.append(f"-DCMAKE_C_COMPILER={spec['mpi'].mpicc}")
- args.append(f"-DCMAKE_CXX_COMPILER={spec['mpi'].mpicxx}")
+
+ args.append(f"-DMPI_C_COMPILER={spec['mpi'].mpicc}")
+ args.append(f"-DMPI_CXX_COMPILER={spec['mpi'].mpicxx}")
args.append(f"-DCMAKE_Fortran_COMPILER={spec['mpi'].mpifc}")
+
+ args.append(self.define_from_variant("ENABLE_METIS", "metis"))
+ args.append(f"-DMETIS_ROOT_DIR={spec['metis'].prefix}")
+
+ args.append(self.define_from_variant("ENABLE_VIZ", "viz"))
+
+ if '+cuda' in spec:
+ args.append("-DENABLE_CUDA=ON")
+ args.append(f"-DCMAKE_CUDA_COMPILER={spec['cuda'].prefix}/bin/nvcc")
+ cuda_arch_vals = spec.variants["cuda_arch"].value
+ if cuda_arch_vals:
+ cuda_arch_sorted = list(sorted(cuda_arch_vals, reverse=True))
+ cuda_arch = cuda_arch_sorted[0]
+ args.append(f"-DCUDA_ARCH={cuda_arch}")
+ else:
+ args.append("-DENABLE_CUDA=OFF")
+
+ if '+rocm' in spec:
+ args.append("-DENABLE_HIP=ON")
+ rocm_arch_vals = spec.variants["amdgpu_target"].value
+ args.append(f"-DROCM_PATH={spec['hip'].prefix}")
+ args.append(f"-DHIP_PATH={spec['hip'].prefix}/hip")
+ if rocm_arch_vals:
+ rocm_arch_sorted = list(sorted(rocm_arch_vals, reverse=True))
+ rocm_arch = rocm_arch_sorted[0]
+ args.append(f"-DROCM_ARCH={rocm_arch}")
+ args.append(f"-DHIP_ARCH={rocm_arch}")
+ else:
+ args.append("-DENABLE_HIP=OFF")
+
+ args.append(self.define_from_variant("ENABLE_OPENMP", "openmp"))
+
+ if '+caliper' in spec:
+ args.append(self.define_from_variant("ENABLE_CALIPER", "caliper"))
+ args.append(f"-Dcaliper_DIR={spec['caliper'].prefix}")
+
+ args.append("-DBUILD_TESTING=OFF")
+ args.append(f"-DN_GROUPS={self.spec.variants['n_groups'].value}")
+
+ args.append(f"-DMPI_CXX_LINK_FLAGS={spec['mpi'].libs.ld_flags}")
+
return args
def install(self, spec, prefix):