From ee6a256eee4e0c00dc46e209c6b20e4e2960c9e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 14:49:16 +0200 Subject: [PATCH] Introduce attribute mode --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 33 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 330 +++++++++++++++++- 2 files changed, 342 insertions(+), 21 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 82373c43b8..bbe888fab5 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -257,6 +257,16 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::string m_originalExtension; + enum class SpecificationVia + { + DefaultValue, + Manually + }; + + ///////////////////// + // Dataset IO mode // + ///////////////////// + enum class IOMode { Dataset, @@ -264,17 +274,28 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl }; IOMode m_mode = IOMode::Dataset; - - enum class SpecificationVia - { - DefaultValue, - Manually - }; SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; std::pair retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + /////////////////////// + // Attribute IO mode // + /////////////////////// + + enum class AttributeMode + { + Short, + Long + }; + + AttributeMode m_attributeMode = AttributeMode::Long; + SpecificationVia m_attributeModeSpecificationVia = + SpecificationVia::DefaultValue; + + std::pair + retrieveAttributeMode(openPMD::json::TracingJSON &config) const; + // HELPER FUNCTIONS // will use the IOHandler to retrieve the correct directory. diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index bc327cdd7e..1a2b5adb7a 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -28,6 +28,7 @@ #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -63,7 +64,8 @@ namespace JSONDefaults { using const_str = char const *const; constexpr const_str openpmd_internal = "__openPMD_internal"; - constexpr const_str IOMode = "IO_mode"; + constexpr const_str IOMode = "dataset_mode"; + constexpr const_str AttributeMode = "attribute_mode"; } // namespace JSONDefaults namespace @@ -200,6 +202,54 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) return std::make_pair(res, res_2); } +auto JSONIOHandlerImpl::retrieveAttributeMode( + openPMD::json::TracingJSON &config) const + -> std::pair +{ + AttributeMode res = m_attributeMode; + SpecificationVia res_2 = SpecificationVia::DefaultValue; + if (auto [configLocation, maybeConfig] = getBackendConfig(config); + maybeConfig.has_value()) + { + auto jsonConfig = maybeConfig.value(); + if (jsonConfig.json().contains("attribute")) + { + auto attributeConfig = jsonConfig["attribute"]; + if (attributeConfig.json().contains("mode")) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + attributeConfig["mode"].json()); + if (!modeOption.has_value()) + { + throw error::BackendConfigSchema( + {configLocation, "mode"}, + "Invalid value of non-string type (accepted values are " + "'dataset' and 'template'."); + } + auto mode = modeOption.value(); + if (mode == "short") + { + res = AttributeMode::Short; + res_2 = SpecificationVia::Manually; + } + else if (mode == "long") + { + res = AttributeMode::Long; + res_2 = SpecificationVia::Manually; + } + else + { + throw error::BackendConfigSchema( + {configLocation, "attribute", "mode"}, + "Invalid value: '" + mode + + "' (accepted values are 'short' and 'long'."); + } + } + } + } + return std::make_pair(res, res_2); +} + std::string JSONIOHandlerImpl::backendConfigKey() const { switch (m_fileFormat) @@ -237,6 +287,8 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_originalExtension{std::move(originalExtension)} { std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie(m_attributeMode, m_attributeModeSpecificationVia) = + retrieveAttributeMode(config); if (auto [_, backendConfig] = getBackendConfig(config); backendConfig.has_value()) @@ -1048,8 +1100,17 @@ void JSONIOHandlerImpl::writeAttribute( } nlohmann::json value; switchType(parameter.dtype, value, parameter.resource); - (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { - {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + switch (m_attributeMode) + { + case AttributeMode::Long: + (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { + {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + break; + case AttributeMode::Short: + // short form + (*jsonVal)[filePosition->id]["attributes"][parameter.name] = value; + break; + } writable->written = true; m_dirty.emplace(file); } @@ -1106,6 +1167,195 @@ void JSONIOHandlerImpl::readDataset( } } +namespace +{ + template + Attribute recoverVectorAttributeFromJson(nlohmann::json const &j) + { + if (!j.is_array()) + { + throw std::runtime_error( + "[JSON backend: recoverVectorAttributeFromJson] Internal " + "control flow error."); + } + + if (j.size() == 7 && + (std::is_same_v || + std::is_same_v || + std::is_same_v)) + { + /* + * The frontend must deal with wrong type reports here. + */ + std::array res; + for (size_t i = 0; i < 7; ++i) + { + res[i] = j[i].get(); + } + return res; + } + else + { + std::vector res; + res.reserve(j.size()); + for (auto const &i : j) + { + res.push_back(i.get()); + } + return res; + } + } + + nlohmann::json::value_t unifyNumericType(nlohmann::json const &j) + { + if (!j.is_array() || j.empty()) + { + throw std::runtime_error( + "[JSON backend: recoverVectorAttributeFromJson] Internal " + "control flow error."); + } + auto dtypeRanking = [](nlohmann::json::value_t dtype) -> unsigned { + switch (dtype) + { + case nlohmann::json::value_t::number_unsigned: + return 0; + case nlohmann::json::value_t::number_integer: + return 1; + case nlohmann::json::value_t::number_float: + return 2; + default: + throw std::runtime_error( + "[JSON backend] Encountered vector with mixed number and " + "non-number datatypes."); + } + }; + auto higherDtype = + [&dtypeRanking]( + nlohmann::json::value_t dt1, + nlohmann::json::value_t dt2) -> nlohmann::json::value_t { + if (dtypeRanking(dt1) > dtypeRanking(dt2)) + { + return dt1; + } + else + { + return dt2; + } + }; + + nlohmann::json::value_t res = j[0].type(); + for (size_t i = 1; i < j.size(); ++i) + { + res = higherDtype(res, j[i].type()); + } + return res; + } + + Attribute recoverAttributeFromJson( + nlohmann::json const &j, std::string const &nameForErrorMessages) + { + // @todo use ReadError once it's mainlined + switch (j.type()) + { + case nlohmann::json::value_t::null: + throw std::runtime_error( + "[JSON backend] Attribute must not be null: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw std::runtime_error( + "[JSON backend] Shorthand-style attribute must not be an " + "object: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + if (j.empty()) + { + std::cerr << "Cannot recover datatype of empty vector without " + "explicit type annotation for attribute '" + << nameForErrorMessages + << "'. Will continue with VEC_INT datatype." + << std::endl; + return std::vector{}; + } + else + { + auto valueType = j[0].type(); + /* + * If the vector is of numeric type, it might happen that the + * first entry is an integer, but a later entry is a float. + * We need to pick the most generic datatype in that case. + */ + if (valueType == nlohmann::json::value_t::number_float || + valueType == nlohmann::json::value_t::number_unsigned || + valueType == nlohmann::json::value_t::number_integer) + { + valueType = unifyNumericType(j); + } + switch (valueType) + { + case nlohmann::json::value_t::null: + throw std::runtime_error( + "[JSON backend] Attribute must not be null: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw std::runtime_error( + "[JSON backend] Invalid contained datatype (object) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + throw std::runtime_error( + "[JSON backend] Invalid contained datatype (array) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::string: + return recoverVectorAttributeFromJson(j); + case nlohmann::json::value_t::boolean: + throw std::runtime_error( + "[JSON backend] Attribute must not be vector of bool: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::number_integer: + return recoverVectorAttributeFromJson< + nlohmann::json::number_integer_t>(j); + case nlohmann::json::value_t::number_unsigned: + return recoverVectorAttributeFromJson< + nlohmann::json::number_unsigned_t>(j); + case nlohmann::json::value_t::number_float: + return recoverVectorAttributeFromJson< + nlohmann::json::number_float_t>(j); + case nlohmann::json::value_t::binary: + throw std::runtime_error( + "[JSON backend] Attribute must not have binary type: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON " + "value."); + } + throw std::runtime_error("Unreachable!"); + } + case nlohmann::json::value_t::string: + return j.get(); + case nlohmann::json::value_t::boolean: + return j.get(); + case nlohmann::json::value_t::number_integer: + return j.get(); + case nlohmann::json::value_t::number_unsigned: + return j.get(); + case nlohmann::json::value_t::number_float: + return j.get(); + case nlohmann::json::value_t::binary: + throw std::runtime_error( + "[JSON backend] Attribute must not have binary type: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON value."); + } + throw std::runtime_error("Unreachable!"); + } +} // namespace + void JSONIOHandlerImpl::readAttribute( Writable *writable, Parameter ¶meters) { @@ -1130,9 +1380,19 @@ void JSONIOHandlerImpl::readAttribute( auto &j = jsonLoc[name]; try { - *parameters.dtype = - Datatype(stringToDatatype(j["datatype"].get())); - switchType(*parameters.dtype, j["value"], parameters); + if (j.is_object()) + { + *parameters.dtype = + Datatype(stringToDatatype(j["datatype"].get())); + switchType( + *parameters.dtype, j["value"], parameters); + } + else + { + Attribute attr = recoverAttributeFromJson(j, name); + *parameters.dtype = attr.dtype; + *parameters.resource = attr.getResource(); + } } catch (json::type_error &) { @@ -1474,7 +1734,10 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) if (res->contains(JSONDefaults::openpmd_internal)) { auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); - if (openpmd_internal.contains(JSONDefaults::IOMode)) + + // Init dataset mode according to file's default + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::IOMode)) { auto modeOption = openPMD::json::asLowerCaseStringDynamic( openpmd_internal.at(JSONDefaults::IOMode)); @@ -1488,17 +1751,42 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) } else if (modeOption.value() == "dataset") { - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) - { - m_mode = IOMode::Dataset; - } + m_mode = IOMode::Dataset; } else if (modeOption.value() == "template") { - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) - { - m_mode = IOMode::Template; - } + m_mode = IOMode::Template; + } + else + { + std::cerr << "[JSON/TOML backend] Warning: Invalid value '" + << modeOption.value() + << "' at internal meta table for entry '" + << JSONDefaults::IOMode + << "'. Will ignore and continue." << std::endl; + } + } + + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::AttributeMode)) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + openpmd_internal.at(JSONDefaults::AttributeMode)); + if (!modeOption.has_value()) + { + std::cerr + << "[JSON/TOML backend] Warning: Invalid value of " + "non-string type at internal meta table for entry '" + << JSONDefaults::AttributeMode + << "'. Will ignore and continue." << std::endl; + } + else if (modeOption.value() == "long") + { + m_attributeMode = AttributeMode::Long; + } + else if (modeOption.value() == "short") + { + m_attributeMode = AttributeMode::Short; } else { @@ -1549,6 +1837,18 @@ void JSONIOHandlerImpl::putJsonContents( break; } + switch (m_attributeMode) + { + case AttributeMode::Short: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::AttributeMode] = "short"; + break; + case AttributeMode::Long: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::AttributeMode] = "long"; + break; + } + switch (m_fileFormat) { case FileFormat::Json: