From 5914373024ece103006dd7f0e5d009a79915e9e5 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 28 Nov 2022 13:07:10 -0500 Subject: [PATCH 1/7] Build ADIOS with SCR (scalable checkpoint restart library) support Co-authored-by: Adam Moody --- CMakeLists.txt | 3 ++- cmake/DetectOptions.cmake | 10 ++++++++++ source/adios2/CMakeLists.txt | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc706eaa17..ca2d53f204 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,6 +156,7 @@ adios_option(MHS "Enable support for MHS" AUTO) adios_option(SST "Enable support for SST" AUTO) adios_option(BP5 "Enable support for BP5" AUTO) adios_option(ZeroMQ "Enable support for ZeroMQ" AUTO) +adios_option(SCR "Enable support for SCR" AUTO) adios_option(HDF5 "Enable support for the HDF5 engine" AUTO) adios_option(IME "Enable support for DDN IME transport" AUTO) adios_option(Python "Enable support for Python bindings" AUTO) @@ -225,7 +226,7 @@ endif() set(ADIOS2_CONFIG_OPTS - BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse + BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME SCR O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index e09b0ea7d5..2b752469fb 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -195,6 +195,16 @@ if(CMAKE_CUDA_COMPILER AND CUDAToolkit_FOUND) set(ADIOS2_HAVE_CUDA TRUE) endif() +# SCR (Scalable Checkpoint/Restart Library) +if(ADIOS2_USE_SCR STREQUAL AUTO) + find_package(SCR) +elseif(ADIOS2_USE_SCR) + find_package(SCR REQUIRED) +endif() +if(SCR_FOUND) + set(ADIOS2_HAVE_SCR TRUE) +endif() + # Fortran if(ADIOS2_USE_Fortran STREQUAL AUTO) include(CheckLanguage) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 4285a6ab7a..1d67024783 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -161,6 +161,10 @@ if(ADIOS2_HAVE_DAOS) target_link_libraries(adios2_core PRIVATE DAOS::DAOS) endif() +if(ADIOS2_HAVE_SCR) + target_link_libraries(adios2_core PRIVATE SCR::SCR) +endif() + if(ADIOS2_HAVE_MPI) add_library(adios2_core_mpi core/IOMPI.cpp From 3587566a345a57ede7644592d304b329f5c48a93 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 28 Nov 2022 13:08:19 -0500 Subject: [PATCH 2/7] Changes on the writer side in the BP engine to be able to run with SCR Co-authored-by: Adam Moody --- source/adios2/engine/bp4/BP4Writer.cpp | 67 ++++++++++++++++++++++---- source/adios2/engine/bp4/BP4Writer.h | 6 +++ 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp index bcb86792b6..f955979c76 100644 --- a/source/adios2/engine/bp4/BP4Writer.cpp +++ b/source/adios2/engine/bp4/BP4Writer.cpp @@ -20,6 +20,10 @@ #include #include +#ifdef ADIOS2_HAVE_SCR +#include "scr.h" +#endif + namespace adios2 { namespace core @@ -201,6 +205,29 @@ void BP4Writer::InitParameters() m_DrainBB = m_WriteToBB && m_BP4Serializer.m_Parameters.BurstBufferDrain; } +std::string SCRRouteFile(std::string name) +{ +#ifdef ADIOS2_HAVE_SCR + char scr_name[SCR_MAX_FILENAME]; + SCR_Route_file(name.c_str(), scr_name); + + std::string s(scr_name); + return s; +#else + return name; +#endif +} + +std::vector AddSCRRouteInfo(const std::vector files) +{ + std::vector newFiles; + for (const auto &name : files) + { + newFiles.push_back(SCRRouteFile(name)); + } + return newFiles; +} + void BP4Writer::InitTransports() { // TODO need to add support for aggregators here later @@ -241,21 +268,28 @@ void BP4Writer::InitTransports() m_BP4Serializer.m_RankMPI); m_FileDrainer.Start(); } + if (m_SCR) + { + m_SubStreamNames = AddSCRRouteInfo(m_SubStreamNames); + } } /* Create the directories either on target or burst buffer if used */ - m_BP4Serializer.m_Profiler.Start("mkdir"); - m_FileDataManager.MkDirsBarrier( - m_SubStreamNames, m_IO.m_TransportsParameters, - m_BP4Serializer.m_Parameters.NodeLocal || m_WriteToBB); - if (m_DrainBB) + if (!m_SCR) { - /* Create the directories on target anyway by main thread */ - m_FileDataManager.MkDirsBarrier(m_DrainSubStreamNames, - m_IO.m_TransportsParameters, - m_BP4Serializer.m_Parameters.NodeLocal); + m_BP4Serializer.m_Profiler.Start("mkdir"); + m_FileDataManager.MkDirsBarrier( + m_SubStreamNames, m_IO.m_TransportsParameters, + m_BP4Serializer.m_Parameters.NodeLocal || m_WriteToBB); + if (m_DrainBB) + { + /* Create the directories on target anyway by main thread */ + m_FileDataManager.MkDirsBarrier( + m_DrainSubStreamNames, m_IO.m_TransportsParameters, + m_BP4Serializer.m_Parameters.NodeLocal); + } + m_BP4Serializer.m_Profiler.Stop("mkdir"); } - m_BP4Serializer.m_Profiler.Stop("mkdir"); if (m_BP4Serializer.m_Aggregator.m_IsAggregator) { @@ -294,6 +328,10 @@ void BP4Writer::InitTransports() m_MetadataFileNames = m_BP4Serializer.GetBPMetadataFileNames(transportsNames); + if (m_SCR) + { + m_MetadataFileNames = AddSCRRouteInfo(m_MetadataFileNames); + } for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i) { @@ -305,6 +343,11 @@ void BP4Writer::InitTransports() m_MetadataIndexFileNames = m_BP4Serializer.GetBPMetadataIndexFileNames(transportsNames); + if (m_SCR) + { + m_MetadataIndexFileNames = + AddSCRRouteInfo(m_MetadataIndexFileNames); + } m_FileMetadataIndexManager.OpenFiles( m_MetadataIndexFileNames, m_OpenMode, m_IO.m_TransportsParameters, @@ -608,6 +651,10 @@ void BP4Writer::WriteProfilingJSONFile() { profileFileName = bpBaseNames[0] + "_profiling.json"; } + if (m_SCR) + { + profileFileName = SCRRouteFile(profileFileName); + } profilingJSONStream.Open(profileFileName, Mode::Write); profilingJSONStream.Write(profilingJSON.data(), profilingJSON.size()); diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h index 056d8d4fa6..093f4f2ab2 100644 --- a/source/adios2/engine/bp4/BP4Writer.h +++ b/source/adios2/engine/bp4/BP4Writer.h @@ -65,6 +65,12 @@ class BP4Writer : public core::Engine /* transport manager for managing the metadata index file */ transportman::TransportMan m_FileMetadataIndexManager; +#ifdef ADIOS2_HAVE_SCR + static constexpr bool m_SCR = true; +#else + static constexpr bool m_SCR = false; +#endif + /* * Burst buffer variables */ From 0a5b131fe0d1ffe69b99ac76322fbe56106ff4a2 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 28 Nov 2022 13:13:39 -0500 Subject: [PATCH 3/7] Adding testing for ADIOS2 with SCR --- testing/adios2/engine/bp/CMakeLists.txt | 7 + .../adios2/engine/bp/TestBPWriteReadSCR.cpp | 1098 +++++++++++++++++ 2 files changed, 1105 insertions(+) create mode 100644 testing/adios2/engine/bp/TestBPWriteReadSCR.cpp diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 9e6a35ae11..ac397ff026 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -115,6 +115,13 @@ bp_gtest_add_tests_helper(WriteReadVector MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW) bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW) +if(ADIOS2_HAVE_SCR) + bp_gtest_add_tests_helper(WriteReadSCR MPI_ALLOW) + foreach(tgt ${Test.Engine.BP.WriteReadSCR-TARGETS}) + target_link_libraries(${tgt} SCR::SCR) + endforeach() +endif() + if(ADIOS2_HAVE_BP5) set(BP5LargeMeta "Engine.BP.BPLargeMetadata.BPWrite1D_LargeMetadata.BP5.Serial") diff --git a/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp b/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp new file mode 100644 index 0000000000..45e6411703 --- /dev/null +++ b/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp @@ -0,0 +1,1098 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include //std::iota +#include + +#include + +#include + +#include "../SmallTestData.h" + +#include "scr.h" + +std::string engineName; // comes from command line +std::string engineParameters; // comes from command line + +class BPWriteReadTestSCR : public ::testing::Test +{ +public: + BPWriteReadTestSCR() = default; + + SmallTestData m_TestData; +}; + +// ADIOS2 BP write and read 1D arrays using SCR +TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("SCRBPWriteRead1D8.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + // Number of steps + const size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + // Write test data using BP + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Test setting parameters + { + io.SetParameter("ProfileUnits", "Microseconds"); + io.SetParameters("Threads=2, CollectiveMetadata = OFF"); + adios2::Params parameters = io.Parameters(); + + auto ProfileUnits = parameters.find("ProfileUnits"); + EXPECT_NE(ProfileUnits, parameters.end()); + EXPECT_EQ(ProfileUnits->second, "Microseconds"); + + auto Threads = parameters.find("Threads"); + EXPECT_NE(Threads, parameters.end()); + EXPECT_EQ(Threads->second, "2"); + + auto CollectiveMetadata = parameters.find("CollectiveMetadata"); + EXPECT_NE(CollectiveMetadata, parameters.end()); + EXPECT_EQ(CollectiveMetadata->second, "OFF"); + + io.ClearParameters(); + + // should not find parameters anymore + parameters = io.Parameters(); + CollectiveMetadata = parameters.find("CollectiveMetadata"); + EXPECT_EQ(CollectiveMetadata, parameters.end()); + } + + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + // auto var_bool = io.DefineVariable("tf", shape, start, + // count); + auto var_char = io.DefineVariable("ch", shape, start, count); + EXPECT_TRUE(var_char); + auto var_iString = io.DefineVariable("iString"); + EXPECT_TRUE(var_iString); + auto var_i8 = io.DefineVariable("i8", shape, start, count); + EXPECT_TRUE(var_i8); + auto var_i16 = + io.DefineVariable("i16", shape, start, count); + EXPECT_TRUE(var_i16); + auto var_i32 = + io.DefineVariable("i32", shape, start, count); + EXPECT_TRUE(var_i32); + auto var_i64 = + io.DefineVariable("i64", shape, start, count); + EXPECT_TRUE(var_i64); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + EXPECT_TRUE(var_u8); + auto var_u16 = + io.DefineVariable("u16", shape, start, count); + EXPECT_TRUE(var_u16); + auto var_u32 = + io.DefineVariable("u32", shape, start, count); + EXPECT_TRUE(var_u32); + auto var_u64 = + io.DefineVariable("u64", shape, start, count); + EXPECT_TRUE(var_u64); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + auto var_r64 = + io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + io.AddTransport("file"); + + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + EXPECT_EQ(bpWriter.OpenMode(), adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + // auto var_bool = io.InquireVariable("bool"); + auto var_char = io.InquireVariable("ch"); + auto var_iString = io.InquireVariable("iString"); + auto var_i8 = io.InquireVariable("i8"); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 1D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({mpiRank * Nx}, {Nx}); + + // var_bool.SetSelection(sel); + var_char.SetSelection(sel); + EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + + // bpWriter.Put(var_bool, currentTestData.TF.data()); + bpWriter.Put(var_char, currentTestData.CHAR.data()); + bpWriter.Put(var_iString, currentTestData.S1); + bpWriter.Put(var_i8, currentTestData.I8.data()); + bpWriter.Put(var_i16, currentTestData.I16.data()); + bpWriter.Put(var_i32, currentTestData.I32.data()); + bpWriter.Put(var_i64, currentTestData.I64.data()); + bpWriter.Put(var_u8, currentTestData.U8.data()); + bpWriter.Put(var_u16, currentTestData.U16.data()); + bpWriter.Put(var_u32, currentTestData.U32.data()); + bpWriter.Put(var_u64, currentTestData.U64.data()); + bpWriter.Put(var_r32, currentTestData.R32.data()); + bpWriter.Put(var_r64, currentTestData.R64.data()); + + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + SCR_Complete_output(scr_valid); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); + + EXPECT_EQ(bpReader.Steps(), NSteps); + + auto var_char = io.InquireVariable("ch"); + EXPECT_TRUE(var_char); + ASSERT_EQ(var_char.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_char.Steps(), NSteps); + ASSERT_EQ(var_char.Shape()[0], mpiSize * Nx); + + auto var_iString = io.InquireVariable("iString"); + EXPECT_TRUE(var_iString); + ASSERT_EQ(var_iString.Shape().size(), 0); + ASSERT_EQ(var_iString.Steps(), NSteps); + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Steps(), NSteps); + ASSERT_EQ(var_i8.Shape()[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Steps(), NSteps); + ASSERT_EQ(var_i16.Shape()[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Steps(), NSteps); + ASSERT_EQ(var_i32.Shape()[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Steps(), NSteps); + ASSERT_EQ(var_i64.Shape()[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Steps(), NSteps); + ASSERT_EQ(var_u8.Shape()[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Steps(), NSteps); + ASSERT_EQ(var_u16.Shape()[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Steps(), NSteps); + ASSERT_EQ(var_u32.Shape()[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Steps(), NSteps); + ASSERT_EQ(var_u64.Shape()[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + + SmallTestData testData; + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + std::array CHAR; + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + + const adios2::Box sel(start, count); + + var_char.SetSelection(sel); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_char.SetStepSelection({t, 1}); + + var_i8.SetStepSelection({t, 1}); + var_i16.SetStepSelection({t, 1}); + var_i32.SetStepSelection({t, 1}); + var_i64.SetStepSelection({t, 1}); + + var_u8.SetStepSelection({t, 1}); + var_u16.SetStepSelection({t, 1}); + var_u32.SetStepSelection({t, 1}); + var_u64.SetStepSelection({t, 1}); + + var_r32.SetStepSelection({t, 1}); + var_r64.SetStepSelection({t, 1}); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + bpReader.Get(var_char, CHAR.data()); + bpReader.Get(var_iString, IString); + + bpReader.Get(var_i8, I8.data()); + bpReader.Get(var_i16, I16.data()); + bpReader.Get(var_i32, I32.data()); + bpReader.Get(var_i64, I64.data()); + + bpReader.Get(var_u8, U8.data()); + bpReader.Get(var_u16, U16.data()); + bpReader.Get(var_u32, U32.data()); + bpReader.Get(var_u64, U64.data()); + + bpReader.Get(var_r32, R32.data()); + bpReader.Get(var_r64, R64.data()); + + bpReader.PerformGets(); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(CHAR[i], currentTestData.CHAR[i]) << msg; + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + SCR_Complete_output(scr_valid); + } +} + +// ADIOS2 BP write and read 2D array using SCR +TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) +{ + // Each process would write a 2x4 array and all processes would + // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here + const std::string fname("SCRBPWriteRead2D2x4Test.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + + // Number of rows + const std::size_t Ny = 2; + + // Number of steps + const std::size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + // Write test data using ADIOS2 + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 2D variables (Ny * (NumOfProcesses * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{Ny, static_cast(Nx * mpiSize)}; + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + auto var_iString = io.DefineVariable("iString"); + EXPECT_TRUE(var_iString); + auto var_i8 = io.DefineVariable("i8", shape, start, count); + EXPECT_TRUE(var_i8); + auto var_i16 = + io.DefineVariable("i16", shape, start, count); + EXPECT_TRUE(var_i16); + auto var_i32 = + io.DefineVariable("i32", shape, start, count); + EXPECT_TRUE(var_i32); + auto var_i64 = + io.DefineVariable("i64", shape, start, count); + EXPECT_TRUE(var_i64); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + EXPECT_TRUE(var_u8); + auto var_u16 = + io.DefineVariable("u16", shape, start, count); + EXPECT_TRUE(var_u16); + auto var_u32 = + io.DefineVariable("u32", shape, start, count); + EXPECT_TRUE(var_u32); + auto var_u64 = + io.DefineVariable("u64", shape, start, count); + EXPECT_TRUE(var_u64); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + auto var_r64 = + io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + io.AddTransport("file"); + + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto var_iString = io.InquireVariable("iString"); + auto var_i8 = io.InquireVariable("i8"); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel( + {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + bpWriter.Put(var_iString, currentTestData.S1); + bpWriter.Put(var_i8, currentTestData.I8.data()); + bpWriter.Put(var_i16, currentTestData.I16.data()); + bpWriter.Put(var_i32, currentTestData.I32.data()); + bpWriter.Put(var_i64, currentTestData.I64.data()); + bpWriter.Put(var_u8, currentTestData.U8.data()); + bpWriter.Put(var_u16, currentTestData.U16.data()); + bpWriter.Put(var_u32, currentTestData.U32.data()); + bpWriter.Put(var_u64, currentTestData.U64.data()); + bpWriter.Put(var_r32, currentTestData.R32.data()); + bpWriter.Put(var_r64, currentTestData.R64.data()); + + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + SCR_Complete_output(scr_valid); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); + + EXPECT_EQ(bpReader.Steps(), NSteps); + auto var_iString = io.InquireVariable("iString"); + EXPECT_TRUE(var_iString); + ASSERT_EQ(var_iString.Shape().size(), 0); + ASSERT_EQ(var_iString.Steps(), NSteps); + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Steps(), NSteps); + ASSERT_EQ(var_i8.Shape()[0], Ny); + ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Steps(), NSteps); + ASSERT_EQ(var_i16.Shape()[0], Ny); + ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Steps(), NSteps); + ASSERT_EQ(var_i32.Shape()[0], Ny); + ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Steps(), NSteps); + ASSERT_EQ(var_i64.Shape()[0], Ny); + ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Steps(), NSteps); + ASSERT_EQ(var_u8.Shape()[0], Ny); + ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Steps(), NSteps); + ASSERT_EQ(var_u16.Shape()[0], Ny); + ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Steps(), NSteps); + ASSERT_EQ(var_u32.Shape()[0], Ny); + ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Steps(), NSteps); + ASSERT_EQ(var_u64.Shape()[0], Ny); + ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + ASSERT_EQ(var_r32.Shape()[0], Ny); + ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], Ny); + ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_i8.SetStepSelection({t, 1}); + var_i16.SetStepSelection({t, 1}); + var_i32.SetStepSelection({t, 1}); + var_i64.SetStepSelection({t, 1}); + + var_u8.SetStepSelection({t, 1}); + var_u16.SetStepSelection({t, 1}); + var_u32.SetStepSelection({t, 1}); + var_u64.SetStepSelection({t, 1}); + + var_r32.SetStepSelection({t, 1}); + var_r64.SetStepSelection({t, 1}); + + bpReader.Get(var_iString, IString); + + bpReader.Get(var_i8, I8.data()); + bpReader.Get(var_i16, I16.data()); + bpReader.Get(var_i32, I32.data()); + bpReader.Get(var_i64, I64.data()); + + bpReader.Get(var_u8, U8.data()); + bpReader.Get(var_u16, U16.data()); + bpReader.Get(var_u32, U32.data()); + bpReader.Get(var_u64, U64.data()); + + bpReader.Get(var_r32, R32.data()); + bpReader.Get(var_r64, R64.data()); + + bpReader.PerformGets(); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + SCR_Complete_output(scr_valid); + } +} + +// ADIOS2 BP write and read 10D arrays using SCR +TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) +{ + // Each process would write a 2x2x...x2 9D array and all processes would + // form a 10D NumberOfProcess x 2 x ... x 2) array + const std::string fname("SCRBPWriteRead10D2x2Test.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of steps + const std::size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + size_t NX = static_cast(mpiSize); + size_t OX = static_cast(mpiRank); + const adios2::Dims shape{NX, 2, 2, 2, 2, 2, 2, 2, 2, 2}; + const adios2::Dims start{OX, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + const adios2::Dims count{1, 2, 2, 2, 2, 2, 2, 2, 2, 2}; + + std::array R64w, R64r; + std::array, 512> CR64w, CR64r; + + // Write test data using ADIOS2 + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 10D variables + { + auto var_r64 = + io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + auto var_c64 = io.DefineVariable>( + "cr64", shape, start, count); + EXPECT_TRUE(var_c64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + io.AddTransport("file"); + + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + double d = mpiRank + 1 + step / 10.0; + // Generate test data for each process uniquely + std::for_each(R64w.begin(), R64w.end(), [&](double &v) { + v = d; + d += 0.0001; + }); + std::for_each(CR64w.begin(), CR64w.end(), + [&](std::complex &v) { + v.real(d); + v.imag(d); + }); + + // Retrieve the variables that previously went out of scope + auto var_r64 = io.InquireVariable("r64"); + auto var_cr64 = io.InquireVariable>("cr64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({start, count}); + var_r64.SetSelection(sel); + var_cr64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + bpWriter.Put(var_r64, R64w.data()); + bpWriter.Put(var_cr64, CR64w.data()); + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + SCR_Complete_output(scr_valid); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); + + EXPECT_EQ(bpReader.Steps(), NSteps); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape().size(), 10); + ASSERT_EQ(var_r64.Shape()[0], NX); + ASSERT_EQ(var_r64.Shape()[1], 2); + ASSERT_EQ(var_r64.Shape()[2], 2); + ASSERT_EQ(var_r64.Shape()[3], 2); + ASSERT_EQ(var_r64.Shape()[4], 2); + ASSERT_EQ(var_r64.Shape()[5], 2); + ASSERT_EQ(var_r64.Shape()[6], 2); + ASSERT_EQ(var_r64.Shape()[7], 2); + ASSERT_EQ(var_r64.Shape()[8], 2); + ASSERT_EQ(var_r64.Shape()[9], 2); + + auto var_cr64 = io.InquireVariable>("cr64"); + EXPECT_TRUE(var_cr64); + ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_cr64.Steps(), NSteps); + ASSERT_EQ(var_cr64.Shape().size(), 10); + ASSERT_EQ(var_cr64.Shape()[0], NX); + ASSERT_EQ(var_cr64.Shape()[1], 2); + ASSERT_EQ(var_cr64.Shape()[2], 2); + ASSERT_EQ(var_cr64.Shape()[3], 2); + ASSERT_EQ(var_cr64.Shape()[4], 2); + ASSERT_EQ(var_cr64.Shape()[5], 2); + ASSERT_EQ(var_cr64.Shape()[6], 2); + ASSERT_EQ(var_cr64.Shape()[7], 2); + ASSERT_EQ(var_cr64.Shape()[8], 2); + ASSERT_EQ(var_cr64.Shape()[9], 2); + + const adios2::Box sel(start, count); + + var_r64.SetSelection(sel); + var_cr64.SetSelection(sel); + + for (size_t step = 0; step < NSteps; ++step) + { + var_r64.SetStepSelection({step, 1}); + var_cr64.SetStepSelection({step, 1}); + bpReader.Get(var_r64, R64r.data()); + bpReader.Get(var_cr64, CR64r.data()); + bpReader.PerformGets(); + + double d = mpiRank + 1 + step / 10.0; + // Re-generate test data for each process uniquely that was written + std::for_each(R64w.begin(), R64w.end(), [&](double &v) { + v = d; + d += 0.0001; + }); + std::for_each(CR64w.begin(), CR64w.end(), + [&](std::complex &v) { + v.real(d); + v.imag(d); + }); + + for (size_t i = 0; i < 512; ++i) + { + std::stringstream ss; + ss << "t=" << step << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(R64r[i], R64w[i]) << msg; + EXPECT_EQ(CR64r[i], CR64w[i]) << msg; + } + } + bpReader.Close(); + SCR_Complete_output(scr_valid); + } +} + +// Test writing a 1D and a 2D array in Deferred mode with SCR +TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) +{ + const std::string fname("SCRBPWriteReadDeferred.bp"); + + int mpiRank = 0, mpiSize = 1; + const size_t Nx = 8; + const std::size_t Ny = 2; + const size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + const adios2::Dims shape_1d{static_cast(Nx * mpiSize)}; + const adios2::Dims start_1d{static_cast(mpiRank * Nx)}; + const adios2::Dims count_1d{Nx}; + const adios2::Dims shape_2d{Ny, static_cast(Nx * mpiSize)}; + const adios2::Dims start_2d{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count_2d{Ny, Nx}; + auto var_1d_i32 = + io.DefineVariable("i32_1d", shape_1d, start_1d, count_1d); + auto var_2d_i32 = + io.DefineVariable("i32_2d", shape_2d, start_2d, count_2d); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + io.SetEngine("BPFile"); + } + io.AddTransport("file"); + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + SmallTestData currentTestData_1d = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + SmallTestData currentTestData_2d = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + + const adios2::Box sel_1d(start_1d, count_1d); + const adios2::Box sel_2d(start_2d, count_2d); + var_1d_i32.SetSelection(sel_1d); + var_2d_i32.SetSelection(sel_2d); + + bpWriter.BeginStep(); + bpWriter.Put(var_1d_i32, currentTestData_1d.I32.data(), + adios2::Mode::Deferred); + bpWriter.Put(var_2d_i32, currentTestData_2d.I32.data(), + adios2::Mode::Deferred); + bpWriter.EndStep(); + } + + bpWriter.Close(); + SCR_Complete_output(scr_valid); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + int scr_valid = 1; + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); + + auto var_1d_i32 = io.InquireVariable("i32_1d"); + ASSERT_EQ(var_1d_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_1d_i32.Steps(), NSteps); + ASSERT_EQ(var_1d_i32.Shape()[0], mpiSize * Nx); + + auto var_2d_i32 = io.InquireVariable("i32_2d"); + ASSERT_EQ(var_2d_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_2d_i32.Steps(), NSteps); + ASSERT_EQ(var_2d_i32.Shape()[0], Ny); + ASSERT_EQ(var_2d_i32.Shape()[1], static_cast(mpiSize * Nx)); + + const adios2::Dims start_1d{static_cast(mpiRank * Nx)}; + const adios2::Dims count_1d{Nx}; + const adios2::Dims start_2d{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count_2d{Ny, Nx}; + + const adios2::Box sel_1d(start_1d, count_1d); + const adios2::Box sel_2d(start_2d, count_2d); + + var_1d_i32.SetSelection(sel_1d); + var_2d_i32.SetSelection(sel_2d); + for (size_t t = 0; t < NSteps; ++t) + { + std::array I32_1d; + std::array I32_2d; + var_2d_i32.SetStepSelection({t, 1}); + var_1d_i32.SetStepSelection({t, 1}); + bpReader.Get(var_2d_i32, I32_1d.data()); + bpReader.Get(var_2d_i32, I32_2d.data()); + bpReader.PerformGets(); + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + EXPECT_EQ(I32_1d[i], currentTestData.I32[i]) << msg; + } + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + EXPECT_EQ(I32_2d[i], currentTestData.I32[i]) << msg; + } + } + bpReader.Close(); + SCR_Complete_output(scr_valid); + } +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + int provided; + + // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); + SCR_Init(); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + if (argc > 2) + { + engineParameters = std::string(argv[2]); + } + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + SCR_Finalize(); + MPI_Finalize(); +#endif + + return result; +} From 73bf6319dd727c484a38d5d3ab59617ca39e3a71 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Tue, 29 Nov 2022 09:35:17 -0500 Subject: [PATCH 4/7] Cmake logic to find the SCR library Co-authored-by: Adam Moody --- cmake/FindSCR.cmake | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 cmake/FindSCR.cmake diff --git a/cmake/FindSCR.cmake b/cmake/FindSCR.cmake new file mode 100644 index 0000000000..f6e48199c2 --- /dev/null +++ b/cmake/FindSCR.cmake @@ -0,0 +1,60 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# +# +# FindSCR +# ----------- +# +# Try to find the SCR library +# +# This module defines the following variables: +# +# SCR_FOUND - System has SCR +# SCR_INCLUDE_DIRS - The SCR include directory +# SCR_LIBRARIES - Link these to use SCR +# +# and the following imported targets: +# SCR::SCR - The SCR library target +# +# You can also set the following variable to help guide the search: +# SCR_ROOT - The install prefix for SCR containing the +# include and lib folders +# Note: this can be set as a CMake variable or an +# environment variable. If specified as a CMake +# variable, it will override any setting specified +# as an environment variable. + +if(NOT SCR_FOUND) + if((NOT SCR_ROOT) AND (NOT (ENV{SCR_ROOT} STREQUAL ""))) + set(SCR_ROOT "$ENV{SCR_ROOT}") + endif() + if(SCR_ROOT) + set(SCR_INCLUDE_OPTS HINTS ${SCR_ROOT}/include NO_DEFAULT_PATHS) + set(SCR_LIBRARY_OPTS + HINTS ${SCR_ROOT}/lib ${SCR_ROOT}/lib64 + NO_DEFAULT_PATHS + ) + endif() + + find_path(SCR_INCLUDE_DIR scr.h ${SCR_INCLUDE_OPTS}) + find_library(SCR_LIBRARY NAMES scr ${SCR_LIBRARY_OPTS}) + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(SCR + FOUND_VAR SCR_FOUND + REQUIRED_VARS SCR_LIBRARY SCR_INCLUDE_DIR + ) + if(SCR_FOUND) + set(SCR_INCLUDE_DIRS ${SCR_INCLUDE_DIR}) + set(SCR_LIBRARIES ${SCR_LIBRARY}) + if(SCR_FOUND AND NOT TARGET SCR::SCR) + add_library(SCR::SCR UNKNOWN IMPORTED) + set_target_properties(SCR::SCR PROPERTIES + IMPORTED_LOCATION "${SCR_LIBRARY}" + INTERFACE_LINK_LIBRARIES "${SCR_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${SCR_INCLUDE_DIR}" + ) + endif() + endif() +endif() From 786c40a4e2523ae82193cd57d25a5cff668a978d Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 30 Nov 2022 14:52:32 -0500 Subject: [PATCH 5/7] Move the SCR initialize and finalize inside the BP4 engine --- examples/hello/bpWriter/helloBPWriter.cpp | 1 + source/adios2/engine/bp4/BP4Writer.cpp | 68 +++++++++++++++-------- source/adios2/engine/bp4/BP4Writer.h | 6 +- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/examples/hello/bpWriter/helloBPWriter.cpp b/examples/hello/bpWriter/helloBPWriter.cpp index 39043656bd..8263180af2 100644 --- a/examples/hello/bpWriter/helloBPWriter.cpp +++ b/examples/hello/bpWriter/helloBPWriter.cpp @@ -54,6 +54,7 @@ int main(int argc, char *argv[]) /*** IO class object: settings and factory of Settings: Variables, * Parameters, Transports, and Execution: Engines */ adios2::IO bpIO = adios.DeclareIO("BPFile_N2N"); + bpIO.SetParameters({{"UseSCR", "1"}}); /** global array : name, { shape (total) }, { start (local) }, { * count diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp index f955979c76..93c3ecf731 100644 --- a/source/adios2/engine/bp4/BP4Writer.cpp +++ b/source/adios2/engine/bp4/BP4Writer.cpp @@ -148,6 +148,42 @@ void BP4Writer::Flush(const int transportIndex) } // PRIVATE +#ifdef ADIOS2_HAVE_SCR +void InitSCR(const std::string fname) +{ + SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); +} + +void CloseSCR(const std::string fname) +{ + int scr_valid = 1; + SCR_Complete_output(scr_valid); +} +#endif + +std::string SCRRouteFile(std::string name) +{ +#ifdef ADIOS2_HAVE_SCR + char scr_name[SCR_MAX_FILENAME]; + SCR_Route_file(name.c_str(), scr_name); + + std::string s(scr_name); + return s; +#else + return name; +#endif +} + +std::vector AddSCRRouteInfo(const std::vector files) +{ + std::vector newFiles; + for (const auto &name : files) + { + newFiles.push_back(SCRRouteFile(name)); + } + return newFiles; +} + void BP4Writer::Init() { InitParameters(); @@ -160,6 +196,10 @@ void BP4Writer::Init() } InitTransports(); InitBPBuffer(); +#ifdef ADIOS2_HAVE_SCR + if (m_SCR) + InitSCR(m_Name); +#endif } #define declare_type(T) \ @@ -203,29 +243,7 @@ void BP4Writer::InitParameters() "in call to BP4::Open to write"); m_WriteToBB = !(m_BP4Serializer.m_Parameters.BurstBufferPath.empty()); m_DrainBB = m_WriteToBB && m_BP4Serializer.m_Parameters.BurstBufferDrain; -} - -std::string SCRRouteFile(std::string name) -{ -#ifdef ADIOS2_HAVE_SCR - char scr_name[SCR_MAX_FILENAME]; - SCR_Route_file(name.c_str(), scr_name); - - std::string s(scr_name); - return s; -#else - return name; -#endif -} - -std::vector AddSCRRouteInfo(const std::vector files) -{ - std::vector newFiles; - for (const auto &name : files) - { - newFiles.push_back(SCRRouteFile(name)); - } - return newFiles; + m_SCR = helper::GetParameter(m_IO.m_Parameters, "UseSCR", m_Verbosity); } void BP4Writer::InitTransports() @@ -584,6 +602,10 @@ void BP4Writer::DoClose(const int transportIndex) m_FileDrainer.Finish(); } // m_BP4Serializer.DeleteBuffers(); +#ifdef ADIOS2_HAVE_SCR + if (m_SCR) + CloseSCR(m_Name); +#endif } void BP4Writer::WriteProfilingJSONFile() diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h index 093f4f2ab2..1fbc5dc44e 100644 --- a/source/adios2/engine/bp4/BP4Writer.h +++ b/source/adios2/engine/bp4/BP4Writer.h @@ -65,11 +65,7 @@ class BP4Writer : public core::Engine /* transport manager for managing the metadata index file */ transportman::TransportMan m_FileMetadataIndexManager; -#ifdef ADIOS2_HAVE_SCR - static constexpr bool m_SCR = true; -#else - static constexpr bool m_SCR = false; -#endif + bool m_SCR; /* * Burst buffer variables From 9f892c32487695c77408bdd78757e376b95ba025 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 30 Nov 2022 14:57:45 -0500 Subject: [PATCH 6/7] Forcing the SCR test to always use MPI --- testing/adios2/engine/bp/CMakeLists.txt | 4 +- .../adios2/engine/bp/TestBPWriteReadSCR.cpp | 42 ++----------------- 2 files changed, 6 insertions(+), 40 deletions(-) diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index ac397ff026..fa5055212a 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -115,8 +115,8 @@ bp_gtest_add_tests_helper(WriteReadVector MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW) bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW) -if(ADIOS2_HAVE_SCR) - bp_gtest_add_tests_helper(WriteReadSCR MPI_ALLOW) +if(ADIOS2_HAVE_SCR AND ADIOS2_HAVE_MPI) + bp_gtest_add_tests_helper(WriteReadSCR MPI_ONLY) foreach(tgt ${Test.Engine.BP.WriteReadSCR-TARGETS}) target_link_libraries(${tgt} SCR::SCR) endforeach() diff --git a/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp b/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp index 45e6411703..f489852460 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp @@ -35,25 +35,17 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) // form a mpiSize * Nx 1D array const std::string fname("SCRBPWriteRead1D8.bp"); - int mpiRank = 0, mpiSize = 1; + int mpiRank, mpiSize; // Number of rows const size_t Nx = 8; // Number of steps const size_t NSteps = 3; - -#if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif // Write test data using BP - -#if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif { adios2::IO io = adios.DeclareIO("TestIO"); @@ -411,7 +403,7 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here const std::string fname("SCRBPWriteRead2D2x4Test.bp"); - int mpiRank = 0, mpiSize = 1; + int mpiRank, mpiSize; // Number of rows const std::size_t Nx = 4; @@ -420,19 +412,11 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) // Number of steps const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif // Write test data using ADIOS2 - -#if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif { adios2::IO io = adios.DeclareIO("TestIO"); @@ -743,14 +727,11 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) // form a 10D NumberOfProcess x 2 x ... x 2) array const std::string fname("SCRBPWriteRead10D2x2Test.bp"); - int mpiRank = 0, mpiSize = 1; + int mpiRank, mpiSize; // Number of steps const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif size_t NX = static_cast(mpiSize); size_t OX = static_cast(mpiRank); @@ -762,12 +743,7 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) std::array, 512> CR64w, CR64r; // Write test data using ADIOS2 - -#if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif { adios2::IO io = adios.DeclareIO("TestIO"); @@ -935,19 +911,15 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) { const std::string fname("SCRBPWriteReadDeferred.bp"); - int mpiRank = 0, mpiSize = 1; + int mpiRank, mpiSize; const size_t Nx = 8; const std::size_t Ny = 2; const size_t NSteps = 3; -#if ADIOS2_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif { adios2::IO io = adios.DeclareIO("TestIO"); const adios2::Dims shape_1d{static_cast(Nx * mpiSize)}; @@ -1068,13 +1040,9 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) int main(int argc, char **argv) { -#if ADIOS2_USE_MPI int provided; - - // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); SCR_Init(); -#endif int result; ::testing::InitGoogleTest(&argc, argv); @@ -1089,10 +1057,8 @@ int main(int argc, char **argv) } result = RUN_ALL_TESTS(); -#if ADIOS2_USE_MPI SCR_Finalize(); MPI_Finalize(); -#endif return result; } From bd77fdaee694366f96d0308e72d52c406ae1a6e3 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Wed, 30 Nov 2022 20:26:18 -0500 Subject: [PATCH 7/7] Update the testing code to the new API --- .../adios2/engine/bp/TestBPWriteReadSCR.cpp | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp b/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp index f489852460..fbdf76e662 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadSCR.cpp @@ -53,6 +53,7 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) { io.SetParameter("ProfileUnits", "Microseconds"); io.SetParameters("Threads=2, CollectiveMetadata = OFF"); + io.SetParameters({{"UseSCR", "1"}}); adios2::Params parameters = io.Parameters(); auto ProfileUnits = parameters.find("ProfileUnits"); @@ -134,8 +135,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) io.AddTransport("file"); - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); EXPECT_EQ(bpWriter.OpenMode(), adios2::Mode::Write); @@ -203,7 +202,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) // Close the file bpWriter.Close(); - SCR_Complete_output(scr_valid); } { @@ -218,8 +216,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) io.SetParameters(engineParameters); } - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadRandomAccess); @@ -392,7 +388,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR1D) } } bpReader.Close(); - SCR_Complete_output(scr_valid); } } @@ -459,6 +454,7 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) EXPECT_TRUE(var_r64); } + io.SetParameters({{"UseSCR", "1"}}); if (!engineName.empty()) { io.SetEngine(engineName); @@ -474,8 +470,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) } io.AddTransport("file"); - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -533,7 +527,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) // Close the file bpWriter.Close(); - SCR_Complete_output(scr_valid); } { @@ -548,8 +541,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) io.SetParameters(engineParameters); } - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadRandomAccess); @@ -716,7 +707,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR2D) } } bpReader.Close(); - SCR_Complete_output(scr_valid); } } @@ -757,6 +747,7 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) EXPECT_TRUE(var_c64); } + io.SetParameters({{"UseSCR", "1"}}); if (!engineName.empty()) { io.SetEngine(engineName); @@ -773,8 +764,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) io.AddTransport("file"); - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -812,7 +801,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) // Close the file bpWriter.Close(); - SCR_Complete_output(scr_valid); } { @@ -827,8 +815,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) io.SetParameters(engineParameters); } - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadRandomAccess); @@ -902,7 +888,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCR10) } } bpReader.Close(); - SCR_Complete_output(scr_valid); } } @@ -933,6 +918,7 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) auto var_2d_i32 = io.DefineVariable("i32_2d", shape_2d, start_2d, count_2d); + io.SetParameters({{"UseSCR", "1"}}); if (!engineName.empty()) { io.SetEngine(engineName); @@ -942,8 +928,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) io.SetEngine("BPFile"); } io.AddTransport("file"); - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -967,7 +951,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) } bpWriter.Close(); - SCR_Complete_output(scr_valid); } { @@ -977,8 +960,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) { io.SetEngine(engineName); } - int scr_valid = 1; - SCR_Start_output(fname.c_str(), SCR_FLAG_CHECKPOINT); adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadRandomAccess); @@ -1030,7 +1011,6 @@ TEST_F(BPWriteReadTestSCR, ADIOS2BPWriteReadSCRDeferred) } } bpReader.Close(); - SCR_Complete_output(scr_valid); } }