diff --git a/CMakeLists.txt b/CMakeLists.txt index bf9972177f..9bd8a73c50 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1363,16 +1363,18 @@ if(openPMD_BUILD_TESTING) ${MPI_TEST_EXE} ${Python_EXECUTABLE} \ ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/data00000100.h5 \ - --outfile ../samples/git-sample/single_iteration.bp && \ + --outfile \ + ../samples/git-sample/single_iteration_%T.bp && \ \ ${MPI_TEST_EXE} ${Python_EXECUTABLE} \ ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/thetaMode/data%T.h5 \ - --outfile ../samples/git-sample/thetaMode/data.bp && \ + --outfile \ + ../samples/git-sample/thetaMode/data_%T.bp && \ \ ${Python_EXECUTABLE} \ ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ - --infile ../samples/git-sample/thetaMode/data.bp \ + --infile ../samples/git-sample/thetaMode/data_%T.bp \ --outfile ../samples/git-sample/thetaMode/data%T.json \ " WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} @@ -1381,17 +1383,17 @@ if(openPMD_BUILD_TESTING) add_test(NAME CLI.pipe.py COMMAND sh -c "${Python_EXECUTABLE} \ - ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ + ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/data%T.h5 \ --outfile ../samples/git-sample/data%T.bp && \ \ ${Python_EXECUTABLE} \ - ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ + ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/thetaMode/data%T.h5 \ --outfile ../samples/git-sample/thetaMode/data%T.bp && \ \ ${Python_EXECUTABLE} \ - ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ + ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/thetaMode/data%T.bp \ --outfile ../samples/git-sample/thetaMode/data%T.json \ " diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 86afffbbdf..d27d62c089 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -229,8 +229,12 @@ class ADIOS2IOHandlerImpl * If the iteration encoding is variableBased, we default to using a group * table, since it is the only reliable way to recover currently active * groups. + * If group-based encoding is used without group table, then + * READ_RANDOM_ACCESS is forbidden as it will be unreliable in reporting + * currently available data. + * Use AbstractIOHandler::m_encoding for this though */ - IterationEncoding m_iterationEncoding = IterationEncoding::groupBased; + // IterationEncoding m_iterationEncoding = IterationEncoding::groupBased; /** * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine */ @@ -434,6 +438,8 @@ namespace ADIOS2Defaults "__openPMD_internal/openPMD2_adios2_schema"; constexpr const_str str_isBoolean = "__is_boolean__"; constexpr const_str str_activeTablePrefix = "__openPMD_groups"; + constexpr const_str str_groupBasedWarning = + "__openPMD_internal/warning_bugprone_groupbased_encoding"; } // namespace ADIOS2Defaults namespace detail @@ -1064,7 +1070,7 @@ namespace detail * without steps. This is not a workaround since not using steps, * while inefficient in ADIOS2, is something that we support. */ - NoStream, + ReadWithoutStream, /** * Rationale behind this state: * When user code opens a Series, series.iterations should contain @@ -1152,8 +1158,8 @@ namespace detail void create_IO(); void configure_IO(ADIOS2IOHandlerImpl &impl); - void configure_IO_Read(std::optional userSpecifiedUsesteps); - void configure_IO_Write(std::optional userSpecifiedUsesteps); + void configure_IO_Read(); + void configure_IO_Write(); }; } // namespace detail diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 1106f78f16..220c8632c3 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" +#include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #if openPMD_HAVE_MPI @@ -168,6 +169,11 @@ namespace internal } } // namespace internal +namespace detail +{ + class BufferedActions; +} + /** Interface for communicating between logical and physically persistent data. * * Input and output operations are channeled through a task queue that is @@ -179,8 +185,12 @@ namespace internal class AbstractIOHandler { friend class Series; + friend class ADIOS2IOHandlerImpl; + friend class detail::BufferedActions; private: + IterationEncoding m_encoding = IterationEncoding::groupBased; + void setIterationEncoding(IterationEncoding encoding) { /* @@ -193,6 +203,8 @@ class AbstractIOHandler // do we really want to have those as const members..? *const_cast(&m_backendAccess) = Access::CREATE; } + + m_encoding = encoding; } public: diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 8ded6f1775..b606412709 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -141,7 +141,6 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; - IterationEncoding encoding = IterationEncoding::groupBased; }; template <> @@ -188,12 +187,6 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; - /* - * The backends might need to ensure availability of certain features - * for some iteration encodings, e.g. availability of ADIOS steps for - * variableBased encoding. - */ - IterationEncoding encoding = IterationEncoding::groupBased; using ParsePreference = internal::ParsePreference; std::shared_ptr out_parsePreference = std::make_shared(ParsePreference::UpFront); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 0737720660..43e29ec98f 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -26,6 +26,7 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Mpi.hpp" @@ -38,6 +39,7 @@ #include #include #include +#include #include #include @@ -526,14 +528,6 @@ void ADIOS2IOHandlerImpl::createFile( if (!writable->written) { - - if (!printedWarningsAlready.noGroupBased && - parameters.encoding == IterationEncoding::groupBased) - { - std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; - printedWarningsAlready.noGroupBased = true; - } - std::string name = parameters.name + fileSuffix(); auto res_pair = getPossiblyExisting(name); @@ -560,7 +554,6 @@ void ADIOS2IOHandlerImpl::createFile( VERIFY(success, "[ADIOS2] Could not create directory."); } - m_iterationEncoding = parameters.encoding; associateWithFile(writable, shared_name); this->m_dirty.emplace(shared_name); @@ -568,7 +561,19 @@ void ADIOS2IOHandlerImpl::createFile( writable->abstractFilePosition = std::make_shared(); // enforce opening the file // lazy opening is deathly in parallel situations - getFileData(shared_name, IfFileNotOpen::OpenImplicitly); + auto &fileData = + getFileData(shared_name, IfFileNotOpen::OpenImplicitly); + + if (!printedWarningsAlready.noGroupBased && + m_handler->m_encoding == IterationEncoding::groupBased) + { + std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; + printedWarningsAlready.noGroupBased = true; + fileData.m_IO.DefineAttribute( + ADIOS2Defaults::str_groupBasedWarning, + std::string("Consider using file-based or variable-based " + "encoding instead.")); + } } } @@ -814,7 +819,6 @@ void ADIOS2IOHandlerImpl::openFile( writable->written = true; writable->abstractFilePosition = std::make_shared(); - m_iterationEncoding = parameters.encoding; // enforce opening the file // lazy opening is deathly in parallel situations auto &fileData = getFileData(file, IfFileNotOpen::OpenImplicitly); @@ -1393,8 +1397,8 @@ void ADIOS2IOHandlerImpl::availableChunks( std::string varName = nameOfVariable(writable); auto engine = ba.getEngine(); // make sure that data are present auto datatype = detail::fromADIOS2Type(ba.m_IO.VariableType(varName)); - bool allSteps = m_handler->m_frontendAccess != Access::READ_LINEAR && - ba.streamStatus == detail::BufferedActions::StreamStatus::NoStream; + bool allSteps = ba.streamStatus == + detail::BufferedActions::StreamStatus::ReadWithoutStream; switchAdios2VariableType( datatype, parameters, @@ -2208,9 +2212,7 @@ namespace detail // might have been closed previously if (engine) { - if (streamStatus == StreamStatus::DuringStep || - (streamStatus == StreamStatus::NoStream && - m_mode == adios2::Mode::Write)) + if (streamStatus == StreamStatus::DuringStep) { engine.EndStep(); } @@ -2307,36 +2309,6 @@ namespace detail } return false; } - - bool useStepsInWriting( - UseGroupTable groupTable, std::string const &engineType) - { - if (engineType == "bp5") - { - /* - * BP5 does not require steps when reading, but it requires - * them when writing. - */ - return true; - } - switch (supportsPerstepParsing(Access::CREATE, engineType)) - { - case PerstepParsing::Required: - return true; - case PerstepParsing::Supported: - switch (groupTable) - { - case UseGroupTable::No: - return false; - case UseGroupTable::Yes: - return true; - } - break; - case PerstepParsing::Unsupported: - return false; - } - return false; // unreachable - } } // namespace size_t BufferedActions::currentStep() @@ -2351,18 +2323,8 @@ namespace detail } } - void BufferedActions::configure_IO_Read( - std::optional userSpecifiedUsesteps) + void BufferedActions::configure_IO_Read() { - if (userSpecifiedUsesteps.has_value() && - m_impl->m_handler->m_backendAccess != Access::READ_WRITE) - { - std::cerr << "Explicitly specified `adios2.usesteps` in Read mode. " - "Usage of steps will be determined by what is found " - "in the file being read." - << std::endl; - } - bool upfrontParsing = supportsUpfrontParsing( m_impl->m_handler->m_backendAccess, m_engineType); PerstepParsing perstepParsing = supportsPerstepParsing( @@ -2389,18 +2351,9 @@ namespace detail m_IO.SetParameter("StreamReader", "On"); break; case PerstepParsing::Unsupported: - streamStatus = StreamStatus::NoStream; - parsePreference = ParsePreference::UpFront; - /* - * Note that in BP4 with linear access mode, we set the - * StreamReader option, disabling upfrontParsing capability. - * So, this branch is only taken by niche engines, such as - * BP3 or HDF5, or by BP5 without group table and normal read - * mode. Need to fall back to random access parsing. - */ -#if openPMD_HAS_ADIOS_2_8 - m_mode = adios2::Mode::ReadRandomAccess; -#endif + throw error::Internal( + "Internal control flow error: Per-Step parsing cannot be " + "unsupported when access type is READ_LINEAR"); break; } break; @@ -2419,7 +2372,7 @@ namespace detail } if (upfrontParsing) { - streamStatus = StreamStatus::NoStream; + streamStatus = StreamStatus::ReadWithoutStream; parsePreference = ParsePreference::UpFront; } else @@ -2442,8 +2395,7 @@ namespace detail } } - void BufferedActions::configure_IO_Write( - std::optional userSpecifiedUsesteps) + void BufferedActions::configure_IO_Write() { optimizeAttributesStreaming = // Also, it should only be done when truly streaming, not @@ -2451,20 +2403,7 @@ namespace detail // streaming engine (otherwise attributes might vanish) nonpersistentEngine(m_engineType); - bool useSteps = useStepsInWriting(useGroupTable(), m_engineType); - if (userSpecifiedUsesteps.has_value()) - { - useSteps = userSpecifiedUsesteps.value(); - if (!useSteps && nonpersistentEngine(m_engineType)) - { - throw error::WrongAPIUsage( - "Cannot switch off IO steps for non-persistent stream " - "engines in ADIOS2."); - } - } - - streamStatus = - useSteps ? StreamStatus::OutsideOfStep : StreamStatus::NoStream; + streamStatus = StreamStatus::OutsideOfStep; } void BufferedActions::configure_IO(ADIOS2IOHandlerImpl &impl) @@ -2479,12 +2418,23 @@ namespace detail #if openPMD_HAS_ADIOS_2_9 if (!m_impl->m_useGroupTable.has_value()) { - switch (m_impl->m_iterationEncoding) + switch (m_impl->m_handler->m_encoding) { + /* + * Writing group-based files with group table might lead to + * datasets that cannot be read with ADIOS2 < v2.9. + * Since we don't really encourage use of group-based encoding, + * but try to give at least a somewhat bug-free experience, + * this is still the best compromise. + */ + case IterationEncoding::groupBased: + /* + * For variable-based encoding, this does not matter as it is + * new and requires >= v2.9 features anyway. + */ case IterationEncoding::variableBased: m_impl->m_useGroupTable = UseGroupTable::Yes; break; - case IterationEncoding::groupBased: case IterationEncoding::fileBased: m_impl->m_useGroupTable = UseGroupTable::No; break; @@ -2494,7 +2444,8 @@ namespace detail if (m_impl->m_modifiableAttributes == ADIOS2IOHandlerImpl::ModifiableAttributes::Unspecified) { - m_impl->m_modifiableAttributes = m_impl->m_iterationEncoding == + m_impl->m_modifiableAttributes = + m_impl->m_handler->m_encoding == IterationEncoding::variableBased ? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes : ADIOS2IOHandlerImpl::ModifiableAttributes::No; @@ -2545,7 +2496,6 @@ namespace detail // set engine parameters std::set alreadyConfigured; - std::optional userSpecifiedUsesteps; bool wasTheFlushTargetSpecifiedViaJSON = false; auto engineConfig = impl.config(ADIOS2Defaults::str_engine); if (!engineConfig.json().is_null()) @@ -2577,8 +2527,10 @@ namespace detail impl.config(ADIOS2Defaults::str_usesteps, engineConfig); if (!_useAdiosSteps.json().is_null() && writeOnly(m_mode)) { - userSpecifiedUsesteps = - std::make_optional(_useAdiosSteps.json().get()); + std::cerr << "[ADIOS2 backend] WARNING: Parameter " + "`adios2.engine.usesteps` is deprecated since use " + "of steps is now always enabled." + << std::endl; } if (engineConfig.json().contains(ADIOS2Defaults::str_flushtarget)) @@ -2621,21 +2573,21 @@ namespace detail { case Access::READ_LINEAR: case Access::READ_ONLY: - configure_IO_Read(userSpecifiedUsesteps); + configure_IO_Read(); break; case Access::READ_WRITE: if (readOnly(m_mode)) { - configure_IO_Read(userSpecifiedUsesteps); + configure_IO_Read(); } else { - configure_IO_Write(userSpecifiedUsesteps); + configure_IO_Write(); } break; case Access::APPEND: case Access::CREATE: - configure_IO_Write(userSpecifiedUsesteps); + configure_IO_Write(); break; } @@ -2806,11 +2758,8 @@ namespace detail // the streaming API was used. m_engine = std::make_optional( adios2::Engine(m_IO.Open(m_file, tempMode))); - if (streamStatus == StreamStatus::NoStream) - { - // Write everything into one big step - m_engine->BeginStep(); - } + m_engine->BeginStep(); + streamStatus = StreamStatus::DuringStep; break; } #if openPMD_HAS_ADIOS_2_8 @@ -2893,6 +2842,41 @@ namespace detail } else { + // If the iteration encoding is group-based and + // no group table is used, we're now at a dead-end. + // Step-by-Step parsing is unreliable in that mode + // since groups might be reported that are not + // there. + // But we were only able to find this out by opening + // the ADIOS2 file with an access mode that was + // possibly wrong, so we would have to close and + // reopen here. + // Since group-based encoding is a bag of trouble in + // ADIOS2 anyway, we just don't support this + // particular use case. + // This failure will only arise when the following + // conditions are met: + // + // 1) group-based encoding + // 2) no group table (i.e. old "ADIOS2 schema") + // 3) LINEAR access mode + // + // This is a relatively lenient restriction other + // than forbidding group-based encoding in ADIOS2 + // altogether. + if (m_impl->m_useGroupTable.value() == + UseGroupTable::No && + m_IO.InquireAttribute( + ADIOS2Defaults::str_groupBasedWarning)) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "Trying to open a group-based ADIOS2 file " + "that does not have a group table with " + "LINEAR access type. That combination is " + "very buggy, so please use " + "READ_ONLY/READ_RANDOM_ACCESS instead."); + } if (!openedANewStep && m_engine.value().BeginStep() != adios2::StepStatus::OK) @@ -2910,11 +2894,11 @@ namespace detail * If openedANewStep is true, then the file consists * of one large step, we just leave it open. */ - streamStatus = StreamStatus::NoStream; + streamStatus = StreamStatus::ReadWithoutStream; } break; } - case StreamStatus::NoStream: + case StreamStatus::ReadWithoutStream: // using random-access mode break; case StreamStatus::DuringStep: @@ -2950,6 +2934,7 @@ namespace detail return m_engine.value(); } + // @todo maybe delete adios2::Engine &BufferedActions::requireActiveStep() { adios2::Engine &eng = getEngine(); @@ -3186,41 +3171,19 @@ namespace detail { if (streamStatus == StreamStatus::Undecided) { - // stream status gets decided on upon opening an engine - getEngine(); + throw error::Internal( + "[BufferedActions::advance()] StreamStatus Undecided before " + "beginning/ending a step?"); } // sic! no else - if (streamStatus == StreamStatus::NoStream) + if (streamStatus == StreamStatus::ReadWithoutStream) { - if (writeOnly(m_mode) && - !m_IO.InquireAttribute( - ADIOS2Defaults::str_usesstepsAttribute)) - { - m_IO.DefineAttribute( - ADIOS2Defaults::str_usesstepsAttribute, 0); - } flush( ADIOS2FlushParams{FlushLevel::UserFlush}, /* writeLatePuts = */ false); return AdvanceStatus::RANDOMACCESS; } - /* - * If advance() is called implicitly (by requireActiveStep()), the - * Series is not necessarily using steps (logically). - * But in some ADIOS2 engines, at least one step must be opened - * (physically) to do anything. - * The usessteps tag should only be set when the Series is *logically* - * using steps. - */ - if (calledExplicitly && writeOnly(m_mode) && - !m_IO.InquireAttribute( - ADIOS2Defaults::str_usesstepsAttribute)) - { - m_IO.DefineAttribute( - ADIOS2Defaults::str_usesstepsAttribute, 1); - } - switch (mode) { case AdvanceMode::ENDSTEP: { @@ -3242,6 +3205,15 @@ namespace detail "opened."); } } + + if (calledExplicitly && writeOnly(m_mode) && + !m_IO.InquireAttribute( + ADIOS2Defaults::str_usesstepsAttribute)) + { + m_IO.DefineAttribute( + ADIOS2Defaults::str_usesstepsAttribute, 1); + } + flush( ADIOS2FlushParams{FlushLevel::UserFlush}, [](BufferedActions &, adios2::Engine &eng) { eng.EndStep(); }, diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index af827704a1..1cd5bc0bd5 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -25,6 +25,7 @@ #include "openPMD/backend/Writable.hpp" #include +#include namespace openPMD { @@ -324,7 +325,22 @@ std::future AbstractIOHandlerImpl::flush() auto ¶meter = deref_dynamic_cast>( i.parameter.get()); writeToStderr( - "[", i.writable->parent, "->", i.writable, "] ADVANCE"); + "[", + i.writable->parent, + "->", + i.writable, + "] ADVANCE ", + [&]() { + switch (parameter.mode) + { + + case AdvanceMode::BEGINSTEP: + return "BEGINSTEP"; + case AdvanceMode::ENDSTEP: + return "ENDSTEP"; + } + throw std::runtime_error("Unreachable!"); + }()); advance(i.writable, parameter); break; } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 6eb0131e70..ce35aee652 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -206,7 +206,6 @@ void Iteration::flushFileBased( /* create file */ Parameter fCreate; fCreate.name = filename; - fCreate.encoding = s.iterationEncoding(); IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); /* create basePath */ diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index 41edd4ed05..326a9221e3 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -74,7 +74,6 @@ void SeriesIterator::initSeriesInLinearReadMode() case IE::variableBased: { Parameter fOpen; fOpen.name = series.get().m_name; - fOpen.encoding = series.iterationEncoding(); series.IOHandler()->enqueue(IOTask(&series, fOpen)); series.IOHandler()->flush(internal::defaultFlushParams); using PP = Parameter::ParsePreference; diff --git a/src/Series.cpp b/src/Series.cpp index dec579675d..deee5d424b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -924,7 +924,6 @@ void Series::flushGorVBased( } Parameter fCreate; fCreate.name = series.m_name; - fCreate.encoding = iterationEncoding(); IOHandler()->enqueue(IOTask(this, fCreate)); } @@ -1004,7 +1003,6 @@ void Series::readFileBased() auto &series = get(); Parameter fOpen; Parameter aRead; - fOpen.encoding = iterationEncoding(); if (!auxiliary::directory_exists(IOHandler()->directory)) throw error::ReadError( @@ -1195,7 +1193,6 @@ void Series::readOneIterationFileBased(std::string const &filePath) Parameter aRead; fOpen.name = filePath; - fOpen.encoding = iterationEncoding(); IOHandler()->enqueue(IOTask(this, fOpen)); IOHandler()->flush(internal::defaultFlushParams); series.iterations.parent() = getWritable(this); @@ -1325,7 +1322,6 @@ auto Series::readGorVBased( auto &series = get(); Parameter fOpen; fOpen.name = series.m_name; - fOpen.encoding = iterationEncoding(); IOHandler()->enqueue(IOTask(this, fOpen)); IOHandler()->flush(internal::defaultFlushParams); series.m_parsePreference = *fOpen.out_parsePreference; @@ -2112,7 +2108,6 @@ void Series::openIteration(IterationIndex_t index, Iteration iteration) auto &series = get(); // open the iteration's file again Parameter fOpen; - fOpen.encoding = iterationEncoding(); fOpen.name = iterationFilename(index); IOHandler()->enqueue(IOTask(this, fOpen)); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index d884766f15..de6c89a39e 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1397,7 +1397,8 @@ void append_mode( std::string const &extension, bool variableBased, ParseMode parseMode, - std::string jsonConfig = "{}") + std::string jsonConfig = "{}", + bool test_read_linear = true) { std::string filename = (variableBased ? "../samples/append/append_variablebased." @@ -1494,6 +1495,7 @@ void append_mode( } }; + if (test_read_linear) { switch (parseMode) { @@ -1618,6 +1620,8 @@ void append_mode( write.flush(); } MPI_Barrier(MPI_COMM_WORLD); + + if (test_read_linear) { Series read(filename, Access::READ_LINEAR, MPI_COMM_WORLD); switch (parseMode) @@ -1712,7 +1716,11 @@ TEST_CASE("append_mode", "[serial]") */ #if HAS_ADIOS_2_8 append_mode( - t, false, ParseMode::LinearWithoutSnapshot, jsonConfigOld); + t, + false, + ParseMode::LinearWithoutSnapshot, + jsonConfigOld, + /* test_read_linear = */ false); #endif #if HAS_ADIOS_2_9 append_mode(t, false, ParseMode::WithSnapshot, jsonConfigNew); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 962ca636aa..b399329eb3 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6613,7 +6613,8 @@ TEST_CASE("chaotic_stream", "[serial]") void unfinished_iteration_test( std::string const &ext, IterationEncoding encoding, - std::string const &config = "{}") + std::string const &config = "{}", + bool test_linear_access = true) { std::cout << "\n\nTESTING " << ext << "\n\n" << std::endl; std::string file = std::string("../samples/unfinished_iteration") + @@ -6702,8 +6703,11 @@ void unfinished_iteration_test( } }; - tryReading(Access::READ_LINEAR); - tryReading(Access::READ_LINEAR, R"({"defer_iteration_parsing": true})"); + if (test_linear_access) + { + tryReading(Access::READ_LINEAR); + tryReading(Access::READ_LINEAR, R"({"defer_iteration_parsing": true})"); + } if (encoding != IterationEncoding::variableBased) { /* @@ -6722,7 +6726,10 @@ TEST_CASE("unfinished_iteration_test", "[serial]") { #if openPMD_HAVE_ADIOS2 unfinished_iteration_test( - "bp", IterationEncoding::groupBased, R"({"backend": "adios2"})"); + "bp", + IterationEncoding::groupBased, + R"({"backend": "adios2"})", + /* test_linear_access = */ false); #if openPMD_HAS_ADIOS_2_9 unfinished_iteration_test( "bp5", @@ -6894,7 +6901,8 @@ void append_mode( std::string const &filename, bool variableBased, ParseMode parseMode, - std::string jsonConfig = "{}") + std::string jsonConfig = "{}", + bool test_read_linear = true) { if (auxiliary::directory_exists("../samples/append")) { @@ -7004,6 +7012,7 @@ void append_mode( } }; + if (test_read_linear) { switch (parseMode) { @@ -7137,6 +7146,7 @@ void append_mode( write.writeIterations(), std::vector{4, 5}); write.flush(); } + if (test_read_linear) { Series read(filename, Access::READ_LINEAR); switch (parseMode) @@ -7225,7 +7235,8 @@ TEST_CASE("append_mode", "[serial]") "../samples/append/append_groupbased." + t, false, ParseMode::LinearWithoutSnapshot, - jsonConfigOld); + jsonConfigOld, + /* test_read_linear = */ false); #if openPMD_HAS_ADIOS_2_9 append_mode( "../samples/append/append_groupbased." + t,