Skip to content

Commit

Permalink
[wip] Make SIO Frame reading (almost) work again
Browse files Browse the repository at this point in the history
  • Loading branch information
@tmadlener
tmadlener committed Mar 22, 2023
1 parent 8f534c4 commit 23406a2
Show file tree
Hide file tree
Showing 11 changed files with 95 additions and 164 deletions.
123 changes: 55 additions & 68 deletions include/podio/AssociationCollection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,6 @@

namespace podio {

template <typename FromT, typename ToT>
std::string associationCollTypeName() {
const static std::string typeName =
std::string("podio::AssociationCollection<") + FromT::TypeName + "," + ToT::TypeName + ">";
return typeName;
}

template <typename FromT, typename ToT>
class AssociationCollection : public podio::CollectionBase {
static_assert(std::is_same_v<FromT, detail::GetDefT<FromT>>,
Expand Down Expand Up @@ -164,28 +157,8 @@ class AssociationCollection : public podio::CollectionBase {
return m_storage.getCollectionBuffers(m_isSubsetColl);
}

// podio::CollectionReadBuffers createBuffers() override /*const*/ {
// // Very cumbersome way at the moment. We get the actual buffers to have the
// // references and vector members sized appropriately (we will use this
// // information to create new buffers outside)
// auto collBuffers = m_storage.getCollectionBuffers(m_isSubsetColl);
// auto readBuffers = podio::CollectionReadBuffers{};
// readBuffers.references = collBuffers.references;
// readBuffers.vectorMembers = collBuffers.vectorMembers;
// readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) {
// CollectionDataT data(buffers, isSubsetColl);
// return std::make_unique<CollectionT>(std::move(data), isSubsetColl);
// };
// readBuffers.recast = [](podio::CollectionReadBuffers& buffers) {
// if (buffers.data) {
// buffers.data = podio::CollectionWriteBuffers::asVector<float>(buffers.data);
// }
// };
// return readBuffers;
// }

std::string getTypeName() const override {
return std::string("podio::AssociationCollection<") + FromT::TypeName + "," + ToT::TypeName + ">";
return podio::detail::associationCollTypeName<FromT, ToT>();
}

std::string getValueTypeName() const override {
Expand Down Expand Up @@ -311,48 +284,62 @@ void to_json(nlohmann::json& j, const AssociationCollection<FromT, ToT>& collect
}
#endif

template <typename FromT, typename ToT>
bool registerAssociationCollection(const std::string& assocTypeName) {
const static auto reg = [&assocTypeName]() {
auto& factory = CollectionBufferFactory::mutInstance();
factory.registerCreationFunc(assocTypeName, AssociationCollection<FromT, ToT>::schemaVersion, [](bool subsetColl) {
auto readBuffers = podio::CollectionReadBuffers{};
readBuffers.data = subsetColl ? nullptr : new AssociationDataContainer();

// Either it is a subset collection or we have two relations
const auto nRefs = subsetColl ? 1 : 2;
readBuffers.references = new podio::CollRefCollection(nRefs);
for (auto& ref : *readBuffers.references) {
// Make sure to place usable buffer pointers here
ref = std::make_unique<std::vector<podio::ObjectID>>();
}

readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) {
AssociationCollectionData<FromT, ToT> data(buffers, isSubsetColl);
return std::make_unique<AssociationCollection<FromT, ToT>>(std::move(data), isSubsetColl);
};

readBuffers.recast = [](podio::CollectionReadBuffers& buffers) {
if (buffers.data) {
buffers.data = podio::CollectionWriteBuffers::asVector<float>(buffers.data);
}
};

return readBuffers;
});
namespace detail {
template <typename FromT, typename ToT>
bool registerAssociationCollection(const std::string& assocTypeName) {
const static auto reg = [&assocTypeName]() {
auto& factory = CollectionBufferFactory::mutInstance();
factory.registerCreationFunc(
assocTypeName, AssociationCollection<FromT, ToT>::schemaVersion, [](bool subsetColl) {
auto readBuffers = podio::CollectionReadBuffers{};
readBuffers.data = subsetColl ? nullptr : new AssociationDataContainer();

// Either it is a subset collection or we have two relations
const auto nRefs = subsetColl ? 1 : 2;
readBuffers.references = new podio::CollRefCollection(nRefs);
for (auto& ref : *readBuffers.references) {
// Make sure to place usable buffer pointers here
ref = std::make_unique<std::vector<podio::ObjectID>>();
}

readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) {
AssociationCollectionData<FromT, ToT> data(buffers, isSubsetColl);
return std::make_unique<AssociationCollection<FromT, ToT>>(std::move(data), isSubsetColl);
};

readBuffers.recast = [](podio::CollectionReadBuffers& buffers) {
if (buffers.data) {
buffers.data = podio::CollectionWriteBuffers::asVector<float>(buffers.data);
}
};

return readBuffers;
});

return true;
}();
return reg;
}
} // namespace detail

return true;
}();
return reg;
}
} // namespace podio

/**
* Main macro for declaring associations. Takes care of the following things: -
* - A type alias with the name TypeName: using TypeAlias =
* AssociationCollection<FromT, ToT>
* - Registering the necessary buffer creation functionality with the
* CollectionBufferFactory.
*
* NOTE: The passed TypeName cannot have a namespace qualifier. If you want the
* type alias to appear in a namespace place the macro call into that namespace.
*
* TODO: Split off the SIOBlock dependency cleanly (i.e. not needing a dedicated
* include, and only present when building with SIO)
*/
#define PODIO_DECLARE_ASSOCIATION(TypeName, FromT, ToT) \
namespace { \
using TypeName = podio::AssociationCollection<FromT, ToT>; \
const auto registerAssociation = \
podio::registerAssociationCollection<FromT, ToT>(podio::associationCollTypeName<FromT, ToT>()); \
} // namespace podio

} // namespace podio
using TypeName = podio::AssociationCollection<FromT, ToT>; \
const static auto REGISTERED_ASSOCIATION_##TypeName = \
podio::detail::registerAssociationCollection<FromT, ToT>(podio::detail::associationCollTypeName<FromT, ToT>());

#endif // PODIO_ASSOCIATIONCOLLECTION_H
9 changes: 8 additions & 1 deletion include/podio/detail/AssociationFwd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,16 @@ namespace detail {
template <typename T>
using GetCollT = typename GetCollType<T>::type;

template <typename FromT, typename ToT>
std::string associationCollTypeName() {
const static std::string typeName =
std::string("podio::AssociationCollection<") + FromT::TypeName + "," + ToT::TypeName + ">";
return typeName;
}

template <typename FromT, typename ToT>
inline std::string associationSIOName() {
auto n = std::string("Association_FROM_") + FromT::TypeName + "_TO_" + FromT::TypeName;
auto n = std::string("ASSOCIATION_FROM_") + FromT::TypeName + "_TO_" + ToT::TypeName;
std::replace(n.begin(), n.end(), ':', '_');
return n;
}
Expand Down
34 changes: 11 additions & 23 deletions include/podio/detail/AssociationSIOBlock.h
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#ifndef PODIO_DETAIL_ASSOCIATIONSIOBLOCK_H
#define PODIO_DETAIL_ASSOCIATIONSIOBLOCK_H

#include "podio/detail/AssociationFwd.h"
#include "podio/AssociationCollection.h"
#include "podio/CollectionBufferFactory.h"

#include "podio/CollectionBuffers.h"
#include "podio/SIOBlock.h"
Expand All @@ -28,17 +29,19 @@ class AssociationSIOBlock : public podio::SIOBlock {
SIOBlock(name, sio::version::encode_version(AssociationCollection<FromT, ToT>::schemaVersion, 0)) {
}

void read(sio::read_device& device, sio::version_type) override {
m_buffers.references->emplace_back(std::make_unique<std::vector<podio::ObjectID>>());
if (!m_subsetColl) {
m_buffers.references->emplace_back(std::make_unique<std::vector<podio::ObjectID>>());
}
void read(sio::read_device& device, sio::version_type version) override {
auto& bufferFactory = podio::CollectionBufferFactory::instance();
// TODO:
// - Error handling of empty optional
auto maybeBuffers = bufferFactory.createBuffers(podio::detail::associationCollTypeName<FromT, ToT>(),
sio::version::major_version(version), m_subsetColl);
m_buffers = maybeBuffers.value();

if (!m_subsetColl) {
unsigned size{0};
device.data(size);
m_buffers.data = new std::vector<float>(size);
auto* dataVec = m_buffers.dataAsVector<float>();
dataVec->resize(size);
podio::handlePODDataSIO(device, dataVec->data(), size);
}

Expand All @@ -60,7 +63,7 @@ class AssociationSIOBlock : public podio::SIOBlock {
podio::handlePODDataSIO(device, dataVec->data(), size);
}

// ---- wirte ref collections ------
// ---- write ref collections ------
auto* refColls = m_buffers.references;
for (auto& refC : *refColls) {
unsigned size = refC->size();
Expand All @@ -69,21 +72,6 @@ class AssociationSIOBlock : public podio::SIOBlock {
}
}

// void createBuffers(const bool subsetCollection = false) override {
// m_subsetColl = subsetCollection;

// m_buffers.references = new podio::CollRefCollection();
// m_buffers.vectorMembers = new podio::VectorMembersInfo();

// m_buffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) {
// AssociationCollectionData<FromT, ToT> data(buffers, isSubsetColl);
// return std::make_unique<AssociationCollection<FromT, ToT>>(std::move(data), isSubsetColl);
// };

// // setCollection(new AssociationCollection<FromT, ToT>());
// // _col->setSubsetCollection(subsetCollection);
// }

SIOBlock* create(const std::string& name) const override {
return new AssociationSIOBlock(name);
}
Expand Down
41 changes: 4 additions & 37 deletions python/podio_class_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from collections.abc import Mapping
from collections import defaultdict

from itertools import zip_longest, combinations_with_replacement, product
from itertools import zip_longest

import jinja2

Expand Down Expand Up @@ -133,11 +133,8 @@ def process(self):

self._write_edm_def_file()

# all possible associations
assoc_combos = self._instantiate_associations()

if 'ROOT' in self.io_handlers:
self._create_selection_xml(assoc_combos)
self._create_selection_xml()

self._write_cmake_lists_file()
self.process_schema_evolution()
Expand Down Expand Up @@ -434,36 +431,6 @@ def _write_edm_def_file(self):
self._write_file('DatamodelDefinition.h',
self._eval_template('DatamodelDefinition.h.jinja2', data))

def _instantiate_associations(self):
"""Instantiate all the associations in a dedicated .cc file and return the
combination of all instantiated things
"""
datatypes = [DataType(d) for d in self.datamodel.datatypes]
includes = [self._build_include_for_class(f'{d.bare_type}Collection', IncludeFrom.INTERNAL) for d in datatypes]

# We want all combinations of our datamodel
combinations = tuple(combinations_with_replacement(datatypes, 2))

if self.upstream_edm:
ext_datatypes = [DataType(d) for d in self.upstream_edm.datatypes]
includes.extend([self._build_include_for_class(f'{d.bare_type}Collection',
IncludeFrom.EXTERNAL) for d in ext_datatypes])

combinations += tuple(product(ext_datatypes, datatypes))

assoc_data = {'package_name': self.package_name,
'includes': includes,
'combinations': combinations}

self._write_file('AssociationsRootDict.h', self._eval_template('AssociationsRootDict.h.jinja2',
assoc_data))

if 'SIO' in self.io_handlers:
self._write_file('AssociationsSIOBlock.cc', self._eval_template('AssociationsSIOBlock.cc.jinja2',
assoc_data))

return combinations

def _get_member_includes(self, members):
"""Process all members and gather the necessary includes"""
includes = set()
Expand Down Expand Up @@ -526,13 +493,13 @@ def _needs_include(self, classname) -> IncludeFrom:

return IncludeFrom.NOWHERE

def _create_selection_xml(self, assoc_combinations):
def _create_selection_xml(self):
"""Create the selection xml that is necessary for ROOT I/O"""
data = {'components': [DataType(c) for c in self.datamodel.components],
'datatypes': [DataType(d) for d in self.datamodel.datatypes],
'old_schema_components': [DataType(d) for d in
self.old_datamodels_datatypes | self.old_datamodels_components],
'associations': assoc_combinations}
}
self._write_file('selection.xml', self._eval_template('selection.xml.jinja2', data))

def _build_include(self, member):
Expand Down
11 changes: 0 additions & 11 deletions python/templates/AssociationsRootDict.h.jinja2
View file

This file was deleted.

12 changes: 0 additions & 12 deletions python/templates/AssociationsSIOBlock.cc.jinja2
View file

This file was deleted.

2 changes: 0 additions & 2 deletions python/templates/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ set(PODIO_TEMPLATES
${CMAKE_CURRENT_LIST_DIR}/SIOBlock.cc.jinja2
${CMAKE_CURRENT_LIST_DIR}/SIOBlock.h.jinja2
${CMAKE_CURRENT_LIST_DIR}/DatamodelDefinition.h.jinja2
${CMAKE_CURRENT_LIST_DIR}/AssociationsRootDict.h.jinja2
${CMAKE_CURRENT_LIST_DIR}/AssociationsSIOBlock.cc.jinja2
${CMAKE_CURRENT_LIST_DIR}/macros/collections.jinja2
${CMAKE_CURRENT_LIST_DIR}/macros/declarations.jinja2
${CMAKE_CURRENT_LIST_DIR}/macros/implementations.jinja2
Expand Down
10 changes: 0 additions & 10 deletions python/templates/selection.xml.jinja2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,6 @@
{%- endif %}
{% endmacro %}

{% macro assoc_selection(from_t, to_t) %}
<class name="podio::Association<{{ from_t.full_type }}, {{ to_t.full_type }}>"/>
<class name="podio::AssociationCollection<{{ from_t.full_type }}, {{ to_t.full_type }}>"/>
{% endmacro %}

<lcgdict>
<selection>

Expand All @@ -33,11 +28,6 @@
<!-- previous schema components and pods -->
{% for class in old_schema_components %}
{{ class_selection(class) }}
{% endfor %}

<!-- associations -->
{% for (from_t, to_t) in associations %}
{{ assoc_selection(from_t, to_t) }}
{% endfor %}

</selection>
Expand Down
4 changes: 4 additions & 0 deletions tests/read_frame_sio.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

#include "read_frame.h"

#include "podio/detail/AssociationSIOBlock.h"

const static auto foo = podio::AssociationSIOBlock<ExampleMC, ex42::ExampleWithARelation>{};

int main() {
return read_frames<podio::SIOFrameReader>("example_frame.sio");
}
8 changes: 8 additions & 0 deletions tests/read_test.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include "podio/AssociationCollection.h"
#include "podio/UserDataCollection.h"
#include "podio/detail/AssociationSIOBlock.h"
#include "podio/podioVersion.h"

// STL
Expand All @@ -34,6 +35,13 @@
// Define an association that is used for the I/O tests
// using TestAssocCollection = podio::AssociationCollection<ExampleMC, ex42::ExampleWithARelation>;
PODIO_DECLARE_ASSOCIATION(TestAssocCollection, ExampleMC, ex42::ExampleWithARelation)
PODIO_DECLARE_ASSOCIATION(TestAssocCollection2, ExampleMC, ExampleHit)

namespace ex42 {
PODIO_DECLARE_ASSOCIATION(AssocInNamespace, ExampleWithARelation, ExampleMC)
}

using A = ex42::AssocInNamespace;

template <typename FixedWidthT>
bool check_fixed_width_value(FixedWidthT actual, FixedWidthT expected, const std::string& type) {
Expand Down
Loading

0 comments on commit 23406a2

Please sign in to comment.