From 23406a2232c37f8b3b8bb607651ddc8c77c65275 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Wed, 22 Mar 2023 18:52:04 +0100 Subject: [PATCH] [wip] Make SIO Frame reading (almost) work again --- include/podio/AssociationCollection.h | 123 ++++++++---------- include/podio/detail/AssociationFwd.h | 9 +- include/podio/detail/AssociationSIOBlock.h | 34 ++--- python/podio_class_generator.py | 41 +----- .../templates/AssociationsRootDict.h.jinja2 | 11 -- .../templates/AssociationsSIOBlock.cc.jinja2 | 12 -- python/templates/CMakeLists.txt | 2 - python/templates/selection.xml.jinja2 | 10 -- tests/read_frame_sio.cpp | 4 + tests/read_test.h | 8 ++ tests/write_frame_sio.cpp | 5 + 11 files changed, 95 insertions(+), 164 deletions(-) delete mode 100644 python/templates/AssociationsRootDict.h.jinja2 delete mode 100644 python/templates/AssociationsSIOBlock.cc.jinja2 diff --git a/include/podio/AssociationCollection.h b/include/podio/AssociationCollection.h index 9fe6b7f99..972e166bd 100644 --- a/include/podio/AssociationCollection.h +++ b/include/podio/AssociationCollection.h @@ -29,13 +29,6 @@ namespace podio { -template -std::string associationCollTypeName() { - const static std::string typeName = - std::string("podio::AssociationCollection<") + FromT::TypeName + "," + ToT::TypeName + ">"; - return typeName; -} - template class AssociationCollection : public podio::CollectionBase { static_assert(std::is_same_v>, @@ -164,28 +157,8 @@ class AssociationCollection : public podio::CollectionBase { return m_storage.getCollectionBuffers(m_isSubsetColl); } - // podio::CollectionReadBuffers createBuffers() override /*const*/ { - // // Very cumbersome way at the moment. We get the actual buffers to have the - // // references and vector members sized appropriately (we will use this - // // information to create new buffers outside) - // auto collBuffers = m_storage.getCollectionBuffers(m_isSubsetColl); - // auto readBuffers = podio::CollectionReadBuffers{}; - // readBuffers.references = collBuffers.references; - // readBuffers.vectorMembers = collBuffers.vectorMembers; - // readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) { - // CollectionDataT data(buffers, isSubsetColl); - // return std::make_unique(std::move(data), isSubsetColl); - // }; - // readBuffers.recast = [](podio::CollectionReadBuffers& buffers) { - // if (buffers.data) { - // buffers.data = podio::CollectionWriteBuffers::asVector(buffers.data); - // } - // }; - // return readBuffers; - // } - std::string getTypeName() const override { - return std::string("podio::AssociationCollection<") + FromT::TypeName + "," + ToT::TypeName + ">"; + return podio::detail::associationCollTypeName(); } std::string getValueTypeName() const override { @@ -311,48 +284,62 @@ void to_json(nlohmann::json& j, const AssociationCollection& collect } #endif -template -bool registerAssociationCollection(const std::string& assocTypeName) { - const static auto reg = [&assocTypeName]() { - auto& factory = CollectionBufferFactory::mutInstance(); - factory.registerCreationFunc(assocTypeName, AssociationCollection::schemaVersion, [](bool subsetColl) { - auto readBuffers = podio::CollectionReadBuffers{}; - readBuffers.data = subsetColl ? nullptr : new AssociationDataContainer(); - - // Either it is a subset collection or we have two relations - const auto nRefs = subsetColl ? 1 : 2; - readBuffers.references = new podio::CollRefCollection(nRefs); - for (auto& ref : *readBuffers.references) { - // Make sure to place usable buffer pointers here - ref = std::make_unique>(); - } - - readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) { - AssociationCollectionData data(buffers, isSubsetColl); - return std::make_unique>(std::move(data), isSubsetColl); - }; - - readBuffers.recast = [](podio::CollectionReadBuffers& buffers) { - if (buffers.data) { - buffers.data = podio::CollectionWriteBuffers::asVector(buffers.data); - } - }; - - return readBuffers; - }); +namespace detail { + template + bool registerAssociationCollection(const std::string& assocTypeName) { + const static auto reg = [&assocTypeName]() { + auto& factory = CollectionBufferFactory::mutInstance(); + factory.registerCreationFunc( + assocTypeName, AssociationCollection::schemaVersion, [](bool subsetColl) { + auto readBuffers = podio::CollectionReadBuffers{}; + readBuffers.data = subsetColl ? nullptr : new AssociationDataContainer(); + + // Either it is a subset collection or we have two relations + const auto nRefs = subsetColl ? 1 : 2; + readBuffers.references = new podio::CollRefCollection(nRefs); + for (auto& ref : *readBuffers.references) { + // Make sure to place usable buffer pointers here + ref = std::make_unique>(); + } + + readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) { + AssociationCollectionData data(buffers, isSubsetColl); + return std::make_unique>(std::move(data), isSubsetColl); + }; + + readBuffers.recast = [](podio::CollectionReadBuffers& buffers) { + if (buffers.data) { + buffers.data = podio::CollectionWriteBuffers::asVector(buffers.data); + } + }; + + return readBuffers; + }); + + return true; + }(); + return reg; + } +} // namespace detail - return true; - }(); - return reg; -} +} // namespace podio +/** + * Main macro for declaring associations. Takes care of the following things: - + * - A type alias with the name TypeName: using TypeAlias = + * AssociationCollection + * - Registering the necessary buffer creation functionality with the + * CollectionBufferFactory. + * + * NOTE: The passed TypeName cannot have a namespace qualifier. If you want the + * type alias to appear in a namespace place the macro call into that namespace. + * + * TODO: Split off the SIOBlock dependency cleanly (i.e. not needing a dedicated + * include, and only present when building with SIO) + */ #define PODIO_DECLARE_ASSOCIATION(TypeName, FromT, ToT) \ - namespace { \ - using TypeName = podio::AssociationCollection; \ - const auto registerAssociation = \ - podio::registerAssociationCollection(podio::associationCollTypeName()); \ - } // namespace podio - -} // namespace podio + using TypeName = podio::AssociationCollection; \ + const static auto REGISTERED_ASSOCIATION_##TypeName = \ + podio::detail::registerAssociationCollection(podio::detail::associationCollTypeName()); #endif // PODIO_ASSOCIATIONCOLLECTION_H diff --git a/include/podio/detail/AssociationFwd.h b/include/podio/detail/AssociationFwd.h index 20d5c9d30..56733cec1 100644 --- a/include/podio/detail/AssociationFwd.h +++ b/include/podio/detail/AssociationFwd.h @@ -36,9 +36,16 @@ namespace detail { template using GetCollT = typename GetCollType::type; + template + std::string associationCollTypeName() { + const static std::string typeName = + std::string("podio::AssociationCollection<") + FromT::TypeName + "," + ToT::TypeName + ">"; + return typeName; + } + template inline std::string associationSIOName() { - auto n = std::string("Association_FROM_") + FromT::TypeName + "_TO_" + FromT::TypeName; + auto n = std::string("ASSOCIATION_FROM_") + FromT::TypeName + "_TO_" + ToT::TypeName; std::replace(n.begin(), n.end(), ':', '_'); return n; } diff --git a/include/podio/detail/AssociationSIOBlock.h b/include/podio/detail/AssociationSIOBlock.h index 4c9a2767d..cc8a2f3ec 100644 --- a/include/podio/detail/AssociationSIOBlock.h +++ b/include/podio/detail/AssociationSIOBlock.h @@ -1,7 +1,8 @@ #ifndef PODIO_DETAIL_ASSOCIATIONSIOBLOCK_H #define PODIO_DETAIL_ASSOCIATIONSIOBLOCK_H -#include "podio/detail/AssociationFwd.h" +#include "podio/AssociationCollection.h" +#include "podio/CollectionBufferFactory.h" #include "podio/CollectionBuffers.h" #include "podio/SIOBlock.h" @@ -28,17 +29,19 @@ class AssociationSIOBlock : public podio::SIOBlock { SIOBlock(name, sio::version::encode_version(AssociationCollection::schemaVersion, 0)) { } - void read(sio::read_device& device, sio::version_type) override { - m_buffers.references->emplace_back(std::make_unique>()); - if (!m_subsetColl) { - m_buffers.references->emplace_back(std::make_unique>()); - } + void read(sio::read_device& device, sio::version_type version) override { + auto& bufferFactory = podio::CollectionBufferFactory::instance(); + // TODO: + // - Error handling of empty optional + auto maybeBuffers = bufferFactory.createBuffers(podio::detail::associationCollTypeName(), + sio::version::major_version(version), m_subsetColl); + m_buffers = maybeBuffers.value(); if (!m_subsetColl) { unsigned size{0}; device.data(size); - m_buffers.data = new std::vector(size); auto* dataVec = m_buffers.dataAsVector(); + dataVec->resize(size); podio::handlePODDataSIO(device, dataVec->data(), size); } @@ -60,7 +63,7 @@ class AssociationSIOBlock : public podio::SIOBlock { podio::handlePODDataSIO(device, dataVec->data(), size); } - // ---- wirte ref collections ------ + // ---- write ref collections ------ auto* refColls = m_buffers.references; for (auto& refC : *refColls) { unsigned size = refC->size(); @@ -69,21 +72,6 @@ class AssociationSIOBlock : public podio::SIOBlock { } } - // void createBuffers(const bool subsetCollection = false) override { - // m_subsetColl = subsetCollection; - - // m_buffers.references = new podio::CollRefCollection(); - // m_buffers.vectorMembers = new podio::VectorMembersInfo(); - - // m_buffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) { - // AssociationCollectionData data(buffers, isSubsetColl); - // return std::make_unique>(std::move(data), isSubsetColl); - // }; - - // // setCollection(new AssociationCollection()); - // // _col->setSubsetCollection(subsetCollection); - // } - SIOBlock* create(const std::string& name) const override { return new AssociationSIOBlock(name); } diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index c9dd3a10f..c4baaba68 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -12,7 +12,7 @@ from collections.abc import Mapping from collections import defaultdict -from itertools import zip_longest, combinations_with_replacement, product +from itertools import zip_longest import jinja2 @@ -133,11 +133,8 @@ def process(self): self._write_edm_def_file() - # all possible associations - assoc_combos = self._instantiate_associations() - if 'ROOT' in self.io_handlers: - self._create_selection_xml(assoc_combos) + self._create_selection_xml() self._write_cmake_lists_file() self.process_schema_evolution() @@ -434,36 +431,6 @@ def _write_edm_def_file(self): self._write_file('DatamodelDefinition.h', self._eval_template('DatamodelDefinition.h.jinja2', data)) - def _instantiate_associations(self): - """Instantiate all the associations in a dedicated .cc file and return the - combination of all instantiated things - """ - datatypes = [DataType(d) for d in self.datamodel.datatypes] - includes = [self._build_include_for_class(f'{d.bare_type}Collection', IncludeFrom.INTERNAL) for d in datatypes] - - # We want all combinations of our datamodel - combinations = tuple(combinations_with_replacement(datatypes, 2)) - - if self.upstream_edm: - ext_datatypes = [DataType(d) for d in self.upstream_edm.datatypes] - includes.extend([self._build_include_for_class(f'{d.bare_type}Collection', - IncludeFrom.EXTERNAL) for d in ext_datatypes]) - - combinations += tuple(product(ext_datatypes, datatypes)) - - assoc_data = {'package_name': self.package_name, - 'includes': includes, - 'combinations': combinations} - - self._write_file('AssociationsRootDict.h', self._eval_template('AssociationsRootDict.h.jinja2', - assoc_data)) - - if 'SIO' in self.io_handlers: - self._write_file('AssociationsSIOBlock.cc', self._eval_template('AssociationsSIOBlock.cc.jinja2', - assoc_data)) - - return combinations - def _get_member_includes(self, members): """Process all members and gather the necessary includes""" includes = set() @@ -526,13 +493,13 @@ def _needs_include(self, classname) -> IncludeFrom: return IncludeFrom.NOWHERE - def _create_selection_xml(self, assoc_combinations): + def _create_selection_xml(self): """Create the selection xml that is necessary for ROOT I/O""" data = {'components': [DataType(c) for c in self.datamodel.components], 'datatypes': [DataType(d) for d in self.datamodel.datatypes], 'old_schema_components': [DataType(d) for d in self.old_datamodels_datatypes | self.old_datamodels_components], - 'associations': assoc_combinations} + } self._write_file('selection.xml', self._eval_template('selection.xml.jinja2', data)) def _build_include(self, member): diff --git a/python/templates/AssociationsRootDict.h.jinja2 b/python/templates/AssociationsRootDict.h.jinja2 deleted file mode 100644 index 8a590de7f..000000000 --- a/python/templates/AssociationsRootDict.h.jinja2 +++ /dev/null @@ -1,11 +0,0 @@ -// AUTOMATICALLY GENERATED FILE - DO NOT EDIT -#include "podio/AssociationCollection.h" - -// Datamodel includes -{% for include in includes %} -{{ include }} -{% endfor %} - -{% for (from_t, to_t) in combinations %} -static podio::AssociationCollection<{{from_t.full_type}}, {{to_t.full_type}}> podio__association_dummy__{{ package_name }}__{{ loop.index0 }}{}; -{% endfor %} diff --git a/python/templates/AssociationsSIOBlock.cc.jinja2 b/python/templates/AssociationsSIOBlock.cc.jinja2 deleted file mode 100644 index 024befdfd..000000000 --- a/python/templates/AssociationsSIOBlock.cc.jinja2 +++ /dev/null @@ -1,12 +0,0 @@ -// AUTOMATICALLY GENERATED FILE - DO NOT EDIT -#include "podio/AssociationCollection.h" -#include "podio/detail/AssociationSIOBlock.h" - -// Datamodel includes -{% for include in includes %} -{{ include }} -{% endfor %} - -{% for (from_t, to_t) in combinations %} -static podio::AssociationSIOBlock<{{from_t.full_type}}, {{to_t.full_type}}> podio__association_dummy__{{ package_name }}__sio_block__{{ loop.index0 }}{}; -{% endfor %} diff --git a/python/templates/CMakeLists.txt b/python/templates/CMakeLists.txt index 8091bb150..be5f4b307 100644 --- a/python/templates/CMakeLists.txt +++ b/python/templates/CMakeLists.txt @@ -15,8 +15,6 @@ set(PODIO_TEMPLATES ${CMAKE_CURRENT_LIST_DIR}/SIOBlock.cc.jinja2 ${CMAKE_CURRENT_LIST_DIR}/SIOBlock.h.jinja2 ${CMAKE_CURRENT_LIST_DIR}/DatamodelDefinition.h.jinja2 - ${CMAKE_CURRENT_LIST_DIR}/AssociationsRootDict.h.jinja2 - ${CMAKE_CURRENT_LIST_DIR}/AssociationsSIOBlock.cc.jinja2 ${CMAKE_CURRENT_LIST_DIR}/macros/collections.jinja2 ${CMAKE_CURRENT_LIST_DIR}/macros/declarations.jinja2 ${CMAKE_CURRENT_LIST_DIR}/macros/implementations.jinja2 diff --git a/python/templates/selection.xml.jinja2 b/python/templates/selection.xml.jinja2 index 6cb4b6841..b0da9ff74 100644 --- a/python/templates/selection.xml.jinja2 +++ b/python/templates/selection.xml.jinja2 @@ -6,11 +6,6 @@ {%- endif %} {% endmacro %} -{% macro assoc_selection(from_t, to_t) %} - - -{% endmacro %} - @@ -33,11 +28,6 @@ {% for class in old_schema_components %} {{ class_selection(class) }} -{% endfor %} - - -{% for (from_t, to_t) in associations %} -{{ assoc_selection(from_t, to_t) }} {% endfor %} diff --git a/tests/read_frame_sio.cpp b/tests/read_frame_sio.cpp index cdc0b8854..501f02b8c 100644 --- a/tests/read_frame_sio.cpp +++ b/tests/read_frame_sio.cpp @@ -2,6 +2,10 @@ #include "read_frame.h" +#include "podio/detail/AssociationSIOBlock.h" + +const static auto foo = podio::AssociationSIOBlock{}; + int main() { return read_frames("example_frame.sio"); } diff --git a/tests/read_test.h b/tests/read_test.h index 9cf8a15d4..e34280304 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -19,6 +19,7 @@ #include "podio/AssociationCollection.h" #include "podio/UserDataCollection.h" +#include "podio/detail/AssociationSIOBlock.h" #include "podio/podioVersion.h" // STL @@ -34,6 +35,13 @@ // Define an association that is used for the I/O tests // using TestAssocCollection = podio::AssociationCollection; PODIO_DECLARE_ASSOCIATION(TestAssocCollection, ExampleMC, ex42::ExampleWithARelation) +PODIO_DECLARE_ASSOCIATION(TestAssocCollection2, ExampleMC, ExampleHit) + +namespace ex42 { +PODIO_DECLARE_ASSOCIATION(AssocInNamespace, ExampleWithARelation, ExampleMC) +} + +using A = ex42::AssocInNamespace; template bool check_fixed_width_value(FixedWidthT actual, FixedWidthT expected, const std::string& type) { diff --git a/tests/write_frame_sio.cpp b/tests/write_frame_sio.cpp index 31df08171..3e1e8c14e 100644 --- a/tests/write_frame_sio.cpp +++ b/tests/write_frame_sio.cpp @@ -1,7 +1,12 @@ +#include "datamodel/ExampleWithARelation.h" #include "write_frame.h" #include "podio/SIOFrameWriter.h" +#include "podio/detail/AssociationSIOBlock.h" + +const static auto foo = podio::AssociationSIOBlock{}; + int main(int, char**) { write_frames("example_frame.sio"); return 0;