[WIP] Extending the BP4 engine to allow data transfers through SCR

CMakeLists.txt

@@ -156,6 +156,7 @@ adios_option(MHS        "Enable support for MHS" AUTO)
 adios_option(SST        "Enable support for SST" AUTO)
 adios_option(BP5        "Enable support for BP5" AUTO)
 adios_option(ZeroMQ     "Enable support for ZeroMQ" AUTO)
+adios_option(SCR        "Enable support for SCR" AUTO)
 adios_option(HDF5       "Enable support for the HDF5 engine" AUTO)
 adios_option(IME        "Enable support for DDN IME transport" AUTO)
 adios_option(Python     "Enable support for Python bindings" AUTO)
@@ -225,7 +226,7 @@ endif()
 
 
 set(ADIOS2_CONFIG_OPTS
-    BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse
+    BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME SCR O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse
 )
 
 GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS})

cmake/DetectOptions.cmake

@@ -195,6 +195,16 @@ if(CMAKE_CUDA_COMPILER AND CUDAToolkit_FOUND)
   set(ADIOS2_HAVE_CUDA TRUE)
 endif()
 
+# SCR (Scalable Checkpoint/Restart Library)
+if(ADIOS2_USE_SCR STREQUAL AUTO)
+  find_package(SCR)
+elseif(ADIOS2_USE_SCR)
+  find_package(SCR REQUIRED)
+endif()
+if(SCR_FOUND)
+  set(ADIOS2_HAVE_SCR TRUE)
+endif()
+
 # Fortran
 if(ADIOS2_USE_Fortran STREQUAL AUTO)
   include(CheckLanguage)

cmake/FindSCR.cmake

@@ -0,0 +1,60 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+#
+# FindSCR
+# -----------
+#
+# Try to find the SCR library
+#
+# This module defines the following variables:
+#
+#   SCR_FOUND        - System has SCR
+#   SCR_INCLUDE_DIRS - The SCR include directory
+#   SCR_LIBRARIES    - Link these to use SCR
+#
+# and the following imported targets:
+#   SCR::SCR - The SCR library target
+#
+# You can also set the following variable to help guide the search:
+#   SCR_ROOT - The install prefix for SCR containing the
+#              include and lib folders
+#              Note: this can be set as a CMake variable or an
+#                    environment variable.  If specified as a CMake
+#                    variable, it will override any setting specified
+#                    as an environment variable.
+
+if(NOT SCR_FOUND)
+  if((NOT SCR_ROOT) AND (NOT (ENV{SCR_ROOT} STREQUAL "")))
+    set(SCR_ROOT "$ENV{SCR_ROOT}")
+  endif()
+  if(SCR_ROOT)
+    set(SCR_INCLUDE_OPTS HINTS ${SCR_ROOT}/include NO_DEFAULT_PATHS)
+    set(SCR_LIBRARY_OPTS
+      HINTS ${SCR_ROOT}/lib ${SCR_ROOT}/lib64
+      NO_DEFAULT_PATHS
+    )
+  endif()
+
+  find_path(SCR_INCLUDE_DIR scr.h ${SCR_INCLUDE_OPTS})
+  find_library(SCR_LIBRARY NAMES scr ${SCR_LIBRARY_OPTS})
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(SCR
+    FOUND_VAR SCR_FOUND
+    REQUIRED_VARS SCR_LIBRARY SCR_INCLUDE_DIR
+  )
+  if(SCR_FOUND)
+    set(SCR_INCLUDE_DIRS ${SCR_INCLUDE_DIR})
+    set(SCR_LIBRARIES ${SCR_LIBRARY})
+    if(SCR_FOUND AND NOT TARGET SCR::SCR)
+      add_library(SCR::SCR UNKNOWN IMPORTED)
+      set_target_properties(SCR::SCR PROPERTIES
+        IMPORTED_LOCATION             "${SCR_LIBRARY}"
+        INTERFACE_LINK_LIBRARIES      "${SCR_LIBRARIES}"
+        INTERFACE_INCLUDE_DIRECTORIES "${SCR_INCLUDE_DIR}"
+      )
+    endif()
+  endif()
+endif()

examples/hello/bpWriter/helloBPWriter.cpp

@@ -54,6 +54,7 @@ int main(int argc, char *argv[])
         /*** IO class object: settings and factory of Settings: Variables,
          * Parameters, Transports, and Execution: Engines */
         adios2::IO bpIO = adios.DeclareIO("BPFile_N2N");
+        bpIO.SetParameters({{"UseSCR", "1"}});
 
         /** global array : name, { shape (total) }, { start (local) }, {
          * count

source/adios2/CMakeLists.txt

@@ -161,6 +161,10 @@ if(ADIOS2_HAVE_DAOS)
   target_link_libraries(adios2_core PRIVATE DAOS::DAOS)
 endif()
 
+if(ADIOS2_HAVE_SCR)
+  target_link_libraries(adios2_core PRIVATE SCR::SCR)
+endif()
+
 if(ADIOS2_HAVE_MPI)
     add_library(adios2_core_mpi
         core/IOMPI.cpp

source/adios2/engine/bp4/BP4Writer.cpp

@@ -20,6 +20,10 @@
 #include <ctime>
 #include <iostream>
 
+#ifdef ADIOS2_HAVE_SCR
+#include "scr.h"
+#endif
+
 namespace adios2
 {
 namespace core
@@ -144,6 +148,42 @@ void BP4Writer::Flush(const int transportIndex)
 }
 
 // PRIVATE
+#ifdef ADIOS2_HAVE_SCR
+void InitSCR(const std::string fname)
+{
+    SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT);
+}
+
+void CloseSCR(const std::string fname)
+{
+    int scr_valid = 1;
+    SCR_Complete_output(scr_valid);
+}
+#endif
+
+std::string SCRRouteFile(std::string name)
+{
+#ifdef ADIOS2_HAVE_SCR
+    char scr_name[SCR_MAX_FILENAME];
+    SCR_Route_file(name.c_str(), scr_name);
+
+    std::string s(scr_name);
+    return s;
+#else
+    return name;
+#endif
+}
+
+std::vector<std::string> AddSCRRouteInfo(const std::vector<std::string> files)
+{
+    std::vector<std::string> newFiles;
+    for (const auto &name : files)
+    {
+        newFiles.push_back(SCRRouteFile(name));
+    }
+    return newFiles;
+}
+
 void BP4Writer::Init()
 {
     InitParameters();
@@ -156,6 +196,10 @@ void BP4Writer::Init()
     }
     InitTransports();
     InitBPBuffer();
+#ifdef ADIOS2_HAVE_SCR
+    if (m_SCR)
+        InitSCR(m_Name);
+#endif
 }
 
 #define declare_type(T)                                                        \
@@ -199,6 +243,7 @@ void BP4Writer::InitParameters()
                                  "in call to BP4::Open to write");
     m_WriteToBB = !(m_BP4Serializer.m_Parameters.BurstBufferPath.empty());
     m_DrainBB = m_WriteToBB && m_BP4Serializer.m_Parameters.BurstBufferDrain;
+    m_SCR = helper::GetParameter(m_IO.m_Parameters, "UseSCR", m_Verbosity);
 }
 
 void BP4Writer::InitTransports()
@@ -241,21 +286,28 @@ void BP4Writer::InitTransports()
                 m_BP4Serializer.m_RankMPI);
             m_FileDrainer.Start();
         }
+        if (m_SCR)
+        {
+            m_SubStreamNames = AddSCRRouteInfo(m_SubStreamNames);
+        }
     }
 
     /* Create the directories either on target or burst buffer if used */
-    m_BP4Serializer.m_Profiler.Start("mkdir");
-    m_FileDataManager.MkDirsBarrier(
-        m_SubStreamNames, m_IO.m_TransportsParameters,
-        m_BP4Serializer.m_Parameters.NodeLocal || m_WriteToBB);
-    if (m_DrainBB)
+    if (!m_SCR)
     {
-        /* Create the directories on target anyway by main thread */
-        m_FileDataManager.MkDirsBarrier(m_DrainSubStreamNames,
-                                        m_IO.m_TransportsParameters,
-                                        m_BP4Serializer.m_Parameters.NodeLocal);
+        m_BP4Serializer.m_Profiler.Start("mkdir");
+        m_FileDataManager.MkDirsBarrier(
+            m_SubStreamNames, m_IO.m_TransportsParameters,
+            m_BP4Serializer.m_Parameters.NodeLocal || m_WriteToBB);
+        if (m_DrainBB)
+        {
+            /* Create the directories on target anyway by main thread */
+            m_FileDataManager.MkDirsBarrier(
+                m_DrainSubStreamNames, m_IO.m_TransportsParameters,
+                m_BP4Serializer.m_Parameters.NodeLocal);
+        }
+        m_BP4Serializer.m_Profiler.Stop("mkdir");
     }
-    m_BP4Serializer.m_Profiler.Stop("mkdir");
 
     if (m_BP4Serializer.m_Aggregator.m_IsAggregator)
     {
@@ -294,6 +346,10 @@ void BP4Writer::InitTransports()
 
         m_MetadataFileNames =
             m_BP4Serializer.GetBPMetadataFileNames(transportsNames);
+        if (m_SCR)
+        {
+            m_MetadataFileNames = AddSCRRouteInfo(m_MetadataFileNames);
+        }
 
         for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i)
         {
@@ -305,6 +361,11 @@ void BP4Writer::InitTransports()
 
         m_MetadataIndexFileNames =
             m_BP4Serializer.GetBPMetadataIndexFileNames(transportsNames);
+        if (m_SCR)
+        {
+            m_MetadataIndexFileNames =
+                AddSCRRouteInfo(m_MetadataIndexFileNames);
+        }
 
         m_FileMetadataIndexManager.OpenFiles(
             m_MetadataIndexFileNames, m_OpenMode, m_IO.m_TransportsParameters,
@@ -541,6 +602,10 @@ void BP4Writer::DoClose(const int transportIndex)
         m_FileDrainer.Finish();
     }
     // m_BP4Serializer.DeleteBuffers();
+#ifdef ADIOS2_HAVE_SCR
+    if (m_SCR)
+        CloseSCR(m_Name);
+#endif
 }
 
 void BP4Writer::WriteProfilingJSONFile()
@@ -608,6 +673,10 @@ void BP4Writer::WriteProfilingJSONFile()
             {
                 profileFileName = bpBaseNames[0] + "_profiling.json";
             }
+            if (m_SCR)
+            {
+                profileFileName = SCRRouteFile(profileFileName);
+            }
             profilingJSONStream.Open(profileFileName, Mode::Write);
             profilingJSONStream.Write(profilingJSON.data(),
                                       profilingJSON.size());

source/adios2/engine/bp4/BP4Writer.h

@@ -65,6 +65,8 @@ class BP4Writer : public core::Engine
     /* transport manager for managing the metadata index file */
     transportman::TransportMan m_FileMetadataIndexManager;
 
+    bool m_SCR;
+
     /*
      *  Burst buffer variables
      */

testing/adios2/engine/bp/CMakeLists.txt

@@ -115,6 +115,13 @@ bp_gtest_add_tests_helper(WriteReadVector MPI_ALLOW)
 bp_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW)
 bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW)
 
+if(ADIOS2_HAVE_SCR AND ADIOS2_HAVE_MPI)
+  bp_gtest_add_tests_helper(WriteReadSCR MPI_ONLY)
+  foreach(tgt ${Test.Engine.BP.WriteReadSCR-TARGETS})
+    target_link_libraries(${tgt} SCR::SCR)
+  endforeach()
+endif()
+
 if(ADIOS2_HAVE_BP5)
   set(BP5LargeMeta "Engine.BP.BPLargeMetadata.BPWrite1D_LargeMetadata.BP5.Serial")