Merge branch 'develop' into cleanup_issues_in_sensei_config

.github/ci/configure_vtkio.cmake

@@ -1,3 +1,2 @@
 set(ENABLE_VTK_IO ON CACHE BOOL "")
 set(ENABLE_VTK_FILTERS ON CACHE BOOL "")
-set(ENABLE_VTK_GENERIC_ARRAYS ON CACHE BOOL "")

CMake/SENSEIConfig.cmake.in

@@ -34,7 +34,6 @@ set(ENABLE_ADIOS1 @ENABLE_ADIOS1@)
 set(ENABLE_ADIOS2 @ENABLE_ADIOS2@)
 set(ENABLE_HDF5 @ENABLE_HDF5@)
 set(ENABLE_CONDUIT @ENABLE_CONDUIT@)
-set(ENABLE_VTK_GENERIC_ARRAYS @ENABLE_VTK_GENERIC_ARRAYS@)
 set(ENABLE_VTK_ACCELERATORS @ENABLE_VTK_ACCELERATORS@)
 set(ENABLE_VTK_MPI @ENABLE_VTK_MPI@)
 set(ENABLE_VTK_IO @ENABLE_VTK_IO@)

CMake/catalyst.cmake

@@ -13,7 +13,7 @@ if (ENABLE_CATALYST)
   if(NOT ParaView_FOUND)
     message(STATUS ${ParaView_NOT_FOUND_MESSAGE})
     message(FATAL_ERROR "Catalyst analysis components require Catalyst build "
-      "(or install directory. Please set ParaView_DIR to point to directory "
+      "(or install directory). Please set ParaView_DIR to point to directory "
       "containing `ParaViewConfig.cmake` or `paraview-config.cmake`.")
   endif()
   if (ParaView_VERSION VERSION_LESS "5.7.0")
@@ -30,12 +30,6 @@ if (ENABLE_CATALYST)
   unset(sensei_pv_components_5_7)
   unset(sensei_pv_components_5_8)
 
-  if(NOT ParaView_FOUND)
-    message(FATAL_ERROR "Catalyst analysis components require Catalyst build "
-      "(or install directory. Please set ParaView_DIR to point to directory "
-      "containing `ParaViewConfig.cmake` or `paraview-config.cmake`.")
-  endif()
-
   add_library(sVTK INTERFACE)
   if (ParaView_VERSION VERSION_LESS "5.7.0")
     target_link_libraries(sVTK INTERFACE ${VTK_LIBRARIES})

CMake/cuda.cmake

@@ -0,0 +1,41 @@
+if(ENABLE_CUDA)
+  include(CheckLanguage)
+  set(CMAKE_CUDA_STANDARD 17)
+  set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+  if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+    set(CMAKE_CUDA_ARCHITECTURES 75)
+  endif()
+  check_language(CUDA)
+if (CMAKE_CUDA_COMPILER)
+  enable_language(CUDA)
+else()
+  message(FATAL_ERROR "CUDA is required by ENABLE_CUDA but CUDA was not found")
+endif()
+
+# Use this function to make sure targets and sources compile on CUDA.
+# arguments:
+
+# TARGET - the name of the target to use CUDA with
+# SOURCES - an optional list of source files that need to be compiled with the
+#           CUDA compiler
+function(sensei_cuda_target)
+  set(OPTS "")
+  set(NVPO TARGET)
+  set(MVO SOURCES)
+  cmake_parse_arguments(PARSE_ARGV 0 CUDA_TGT "${OPTS}" "${NVPO}" "${MVO}")
+
+  message(STATUS "Created CUDA target ${CUDA_TGT_TARGET}")
+
+  target_compile_features(${CUDA_TGT_TARGET} PUBLIC cxx_std_17)
+
+  set_target_properties(${CUDA_TGT_TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+  set_target_properties(${CUDA_TGT_TARGET} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+  set_target_properties(${CUDA_TGT_TARGET} PROPERTIES CUDA_ARCHITECTURES "75")
+
+  if (CUDA_TGT_SOURCES)
+     message(STATUS "Compiling ${CUDA_TGT_SOURCES} with the CUDA compiler")
+     set_source_files_properties(${CUDA_TGT_SOURCES} PROPERTIES LANGUAGE CUDA)
+  endif()
+endfunction()
+
+endif()

CMake/options.cmake

@@ -1,5 +1,9 @@
 option(ENABLE_SENSEI "Enable Sensei infrastucture" ON)
 
+cmake_dependent_option(ENABLE_CUDA
+  "Enable analysis methods that use CUDA" OFF
+  "ENABLE_SENSEI" OFF)
+
 cmake_dependent_option(ENABLE_PYTHON
   "Enable Python binding to Sensei infrastucture" OFF
   "ENABLE_SENSEI" OFF)
@@ -40,10 +44,6 @@ cmake_dependent_option(ENABLE_LIBSIM
   "Enable analysis methods that use Libsim" OFF
   "ENABLE_SENSEI" OFF)
 
-cmake_dependent_option(ENABLE_VTK_GENERIC_ARRAYS
-  "VTK build has Generic arrays" OFF
-  "ENABLE_SENSEI" OFF)
-
 cmake_dependent_option(ENABLE_VTK_IO
   "Enable use of vtk I/O" OFF
   "ENABLE_SENSEI" OFF)
@@ -82,6 +82,7 @@ option(ENABLE_KRIPKE "Enable Kripke miniapp (experimental)" OFF)
 option(SENSEI_USE_EXTERNAL_pugixml "Use external pugixml library" OFF)
 
 message(STATUS "ENABLE_SENSEI=${ENABLE_SENSEI}")
+message(STATUS "ENABLE_CUDA=${ENABLE_CUDA}")
 message(STATUS "ENABLE_PYTHON=${ENABLE_PYTHON}")
 message(STATUS "ENABLE_CATALYST=${ENABLE_CATALYST}")
 message(STATUS "ENABLE_CATALYST_PYTHON=${ENABLE_CATALYST}")
@@ -91,7 +92,6 @@ message(STATUS "ENABLE_HDF5=${ENABLE_HDF5}")
 message(STATUS "ENABLE_CONDUIT=${ENABLE_CONDUIT}")
 message(STATUS "ENABLE_ASCENT=${ENABLE_ASCENT}")
 message(STATUS "ENABLE_LIBSIM=${ENABLE_LIBSIM}")
-message(STATUS "ENABLE_VTK_GENERIC_ARRAYS=${ENABLE_VTK_GENERIC_ARRAYS}")
 message(STATUS "ENABLE_VTK_IO=${ENABLE_VTK_IO}")
 message(STATUS "ENABLE_VTK_MPI=${ENABLE_VTK_MPI}")
 message(STATUS "ENABLE_VTK_RENDERING=${ENABLE_VTK_RENDERING}")

CMakeLists.txt

@@ -8,6 +8,7 @@ include(CMakeDependentOption)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake")
 
 include(options)
+include(cuda)
 include(build)
 include(threads)
 include(mpi)

README.md

@@ -102,8 +102,8 @@ $ make install
 ### Build Options
 | Build Option | Default | Description                     |
 |--------------|---------|---------------------------------|
+| `ENABLE_CUDA` | OFF | Enables CUDA accelerated codes. Requires compute capability 7.5 and CUDA 11 or later. |
 | `ENABLE_PYTHON` | OFF | Enables Python bindings. Requires VTK, Python, Numpy, mpi4py, and SWIG. |
-| `ENABLE_VTK_GENERIC_ARRAYS` | OFF | Enables use of VTK's generic array feature.  |
 | `ENABLE_CATALYST` | OFF | Enables the Catalyst analysis adaptor. Depends on ParaView Catalyst. Set `ParaView_DIR`. |
 | `ENABLE_CATALYST_PYTHON` | OFF | Enables Python features of the Catalyst analysis adaptor.  |
 | `ENABLE_ASCENT` | OFF | Enables the Ascent analysis adaptor. |

miniapps/oscillators/Block.cpp

@@ -1,4 +1,5 @@
 #include "Block.h"
+#include "BlockInternals.h"
 
 // --------------------------------------------------------------------------
 void Block::update_fields(float t)
@@ -8,7 +9,6 @@ void Block::update_fields(float t)
     int ni = shape[0];
     int nj = shape[1];
     int nk = shape[2];
-    int nij = ni*nj;
     int i0 = bounds.min[0];
     int j0 = bounds.min[1];
     int k0 = bounds.min[2];
@@ -19,31 +19,29 @@ void Block::update_fields(float t)
     float y0 = origin[1] + dy;
     float z0 = origin[2] + dz;
     float *pdata = grid.data();
-    for (int k = 0; k < nk; ++k)
-    {
-        float z = z0 + dz*(k0 + k);
-        float *pdk = pdata + k*nij;
-        for (int j = 0; j < nj; ++j)
-        {
-            float y = y0 + dy*(j0 + j);
-            float *pd = pdk + j*ni;
-            for (int i = 0; i < ni; ++i)
-            {
-                float x = x0 + dx*(i0 + i);
-                pd[i] = 0.f;
-                for (auto& o : oscillators)
-                    pd[i] += o.evaluate({x,y,z}, t);
-            }
-        }
-    }
 
+    // TODO -- run time GPU/CPU load balancing.
+#if defined(OSCILLATOR_CUDA)
+    int deviceId = 0;
+#else
+    int deviceId = -1;
+#endif
+    BlockInternals::UpdateFields(deviceId, t, oscillators.get(),
+        nOscillators, ni,nj,nk, i0,j0,k0, x0,y0,z0, dx,dy,dz,
+        pdata);
+}
+
+// --------------------------------------------------------------------------
+void Block::update_particles(float t)
+{
     // update the velocity field on the particle mesh
+    const Oscillator *pOsc = oscillators.get();
     for (auto& particle : particles)
     {
         particle.velocity = { 0, 0, 0 };
-        for (auto& o : oscillators)
+        for (unsigned long q = 0; q < nOscillators; ++q)
         {
-            particle.velocity += o.evaluateGradient(particle.position, t);
+            particle.velocity += pOsc[q].evaluateGradient(particle.position, t);
         }
         // scale the gradient to get "units" right for velocity
         particle.velocity *= velocity_scale;

miniapps/oscillators/Block.h

@@ -4,31 +4,36 @@
 #include <sdiy/master.hpp>
 #include <sdiy/decomposition.hpp>
 #include <sdiy/io/bov.hpp>
-#include <sdiy/grid.hpp>
 #include <sdiy/vertices.hpp>
+#include <sdiy/point.hpp>
 
 #include "Oscillator.h"
 #include "Particles.h"
+#include "Grid.h"
 
 #include <vector>
 #include <ostream>
 
 struct Block
 {
-    using Vertex = sdiy::Grid<float,3>::Vertex;
+    using Vertex = oscillator::Grid<float,3>::Vertex;
 
      Block(int gid_, const sdiy::DiscreteBounds& bounds_, const sdiy::DiscreteBounds& domain_,
          const sdiy::Point<float,3> &origin_, const sdiy::Point<float,3> &spacing_,
-         int nghost_, const std::vector<Oscillator>& oscillators_, float velocity_scale_) :
+         int nghost_, const std::shared_ptr<const Oscillator> &oscillators_, unsigned long nOscillators_,
+         float velocity_scale_) :
                 gid(gid_), velocity_scale(velocity_scale_), bounds(bounds_),
                 domain(domain_), origin(origin_), spacing(spacing_), nghost(nghost_),
                 grid(Vertex(&bounds.max[0]) - Vertex(&bounds.min[0]) + Vertex::one()),
-                oscillators(oscillators_)
+                oscillators(oscillators_), nOscillators(nOscillators_)
     {}
 
-    // update scalar and vector fields
+    // update mesh based scalar and vector fields
     void update_fields(float t);
 
+    // update particle based scalar and vector fields
+    void update_particles(float t);
+
     // update pareticle positions
     void move_particles(float dt, const sdiy::Master::ProxyWithLink& cp);
 
@@ -39,16 +44,18 @@ struct Block
     static void *create(){ return new Block; }
     static void destroy(void* b){ delete static_cast<Block*>(b); }
 
-    int                             gid;    // block id
-    float                           velocity_scale;
-    sdiy::DiscreteBounds             bounds; // cell centered index space dimensions of this block
-    sdiy::DiscreteBounds             domain; // cell centered index space dimensions of the computational domain
-    sdiy::Point<float,3>             origin;  // lower left most corner in the computational domain
-    sdiy::Point<float,3>             spacing; // mesh spacing
-    int                             nghost; // number of ghost zones
-    sdiy::Grid<float,3>              grid;   // container for the gridded data arrays
-    std::vector<Particle>           particles;
-    std::vector<Oscillator>         oscillators;
+    int                               gid;    // block id
+    float                             velocity_scale;
+    sdiy::DiscreteBounds              bounds; // cell centered index space dimensions of this block
+    sdiy::DiscreteBounds              domain; // cell centered index space dimensions of the computational domain
+    sdiy::Point<float,3>              origin;  // lower left most corner in the computational domain
+    sdiy::Point<float,3>              spacing; // mesh spacing
+    int                               nghost; // number of ghost zones
+    oscillator::Grid<float,3>         grid;   // container for the gridded data arrays
+    std::vector<Particle>             particles;
+
+    std::shared_ptr<const Oscillator> oscillators; // a pointer to the list of oscillators
+    unsigned long                     nOscillators; // number of oscillators
 
  private:
     // for create; to let Master manage the blocks