From e9a7982d47acc860b89d0e193602847c0c3d403e Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Fri, 21 Apr 2023 10:42:16 +0200 Subject: [PATCH 01/24] add new frame based podio interface prepare compatibility for new frame based Podio interface drop old features of storing simple types automatically in ROOT temporarily remove feature for access to collection metadata --- k4FWCore/CMakeLists.txt | 6 +- k4FWCore/components/PodioInput.cpp | 13 +- k4FWCore/components/PodioLegacyInput.cpp | 53 +++++ k4FWCore/components/PodioLegacyInput.h | 41 ++++ k4FWCore/components/PodioLegacyOutput.cpp | 200 ++++++++++++++++++ k4FWCore/components/PodioLegacyOutput.h | 59 ++++++ k4FWCore/components/PodioOutput.cpp | 139 ++---------- k4FWCore/components/PodioOutput.h | 17 +- k4FWCore/components/k4LegacyDataSvc.cpp | 16 ++ k4FWCore/components/k4LegacyDataSvc.h | 15 ++ k4FWCore/components/k4TestFloatProducer.cpp | 35 --- k4FWCore/components/k4TestFloatProducer.h | 28 --- k4FWCore/include/k4FWCore/DataHandle.h | 45 ++-- k4FWCore/include/k4FWCore/DataWrapper.h | 8 +- k4FWCore/include/k4FWCore/PodioDataSvc.h | 31 ++- .../include/k4FWCore/PodioLegacyDataSvc.h | 82 +++++++ k4FWCore/src/PodioDataSvc.cpp | 68 +++--- k4FWCore/src/PodioLegacyDataSvc.cpp | 126 +++++++++++ k4FWCore/test/options/simple_write.py | 23 -- test/k4FWCoreTest/CMakeLists.txt | 5 + test/k4FWCoreTest/options/TwoProducers.py | 2 - .../options/createLegacyExampleEventData.py | 23 ++ .../options/readLegacyExampleEventData.py | 24 +++ .../k4FWCoreTest_AlgorithmWithTFile.cpp | 8 +- .../k4FWCoreTest_AlgorithmWithTFile.h | 6 +- .../k4FWCoreTest_CreateExampleEventData.cpp | 11 +- .../k4FWCoreTest_CreateExampleEventData.h | 7 +- .../components/k4FWCoreTest_cellID_reader.cpp | 12 +- .../components/k4FWCoreTest_cellID_writer.cpp | 5 +- 29 files changed, 766 insertions(+), 342 deletions(-) create mode 100644 k4FWCore/components/PodioLegacyInput.cpp create mode 100644 k4FWCore/components/PodioLegacyInput.h create mode 100644 k4FWCore/components/PodioLegacyOutput.cpp create mode 100644 k4FWCore/components/PodioLegacyOutput.h create mode 100644 k4FWCore/components/k4LegacyDataSvc.cpp create mode 100644 k4FWCore/components/k4LegacyDataSvc.h delete mode 100644 k4FWCore/components/k4TestFloatProducer.cpp delete mode 100644 k4FWCore/components/k4TestFloatProducer.h create mode 100644 k4FWCore/include/k4FWCore/PodioLegacyDataSvc.h create mode 100644 k4FWCore/src/PodioLegacyDataSvc.cpp delete mode 100644 k4FWCore/test/options/simple_write.py create mode 100644 test/k4FWCoreTest/options/createLegacyExampleEventData.py create mode 100644 test/k4FWCoreTest/options/readLegacyExampleEventData.py diff --git a/k4FWCore/CMakeLists.txt b/k4FWCore/CMakeLists.txt index 3142ee14..0b3535a6 100644 --- a/k4FWCore/CMakeLists.txt +++ b/k4FWCore/CMakeLists.txt @@ -9,6 +9,7 @@ gaudi_install(SCRIPTS) gaudi_add_library(k4FWCore SOURCES src/PodioDataSvc.cpp + src/PodioLegacyDataSvc.cpp src/KeepDropSwitch.cpp LINK Gaudi::GaudiAlgLib Gaudi::GaudiKernel podio::podioRootIO ROOT::Core ROOT::RIO ROOT::Tree ) @@ -30,8 +31,3 @@ install(TARGETS k4FWCore k4FWCorePlugins LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT shlib COMPONENT dev) - -add_test(NAME WriteTest - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - COMMAND ${CMAKE_BINARY_DIR}/run ${PROJECT_SOURCE_DIR}/k4FWCore/scripts/k4run k4FWCore/test/options/simple_write.py) - diff --git a/k4FWCore/components/PodioInput.cpp b/k4FWCore/components/PodioInput.cpp index 6d2f9081..5df428ac 100644 --- a/k4FWCore/components/PodioInput.cpp +++ b/k4FWCore/components/PodioInput.cpp @@ -19,15 +19,8 @@ StatusCode PodioInput::initialize() { if (nullptr == m_podioDataSvc) return StatusCode::FAILURE; - auto idTable = m_podioDataSvc->getCollectionIDs(); - for (auto& name : m_collectionNames) { - debug() << "Finding collection " << name << " in collection registry." << endmsg; - if (!idTable->present(name)) { - error() << "Requested product " << name << " not found." << endmsg; - return StatusCode::FAILURE; - } - m_collectionIDs.push_back(idTable->collectionID(name)); - } + // TODO: add an upfront check for existence of data products + return StatusCode::SUCCESS; } @@ -37,7 +30,7 @@ StatusCode PodioInput::execute() { for (auto& id : m_collectionIDs) { const std::string& collName = m_collectionNames.value().at(cntr++); debug() << "Registering collection to read " << collName << " with id " << id << endmsg; - if (m_podioDataSvc->readCollection(collName, id).isFailure()) { + if (m_podioDataSvc->readCollection(collName).isFailure()) { return StatusCode::FAILURE; } } diff --git a/k4FWCore/components/PodioLegacyInput.cpp b/k4FWCore/components/PodioLegacyInput.cpp new file mode 100644 index 00000000..e905431d --- /dev/null +++ b/k4FWCore/components/PodioLegacyInput.cpp @@ -0,0 +1,53 @@ +#include "PodioLegacyInput.h" + +#include "TFile.h" +#include "TROOT.h" + +#include "k4FWCore/DataWrapper.h" +#include "k4FWCore/PodioLegacyDataSvc.h" + +DECLARE_COMPONENT(PodioLegacyInput) + +PodioLegacyInput::PodioLegacyInput(const std::string& name, ISvcLocator* svcLoc) : GaudiAlgorithm(name, svcLoc) {} + +StatusCode PodioLegacyInput::initialize() { + if (GaudiAlgorithm::initialize().isFailure()) + return StatusCode::FAILURE; + + // check whether we have the PodioEvtSvc active + m_podioLegacyDataSvc = dynamic_cast(evtSvc().get()); + if (nullptr == m_podioLegacyDataSvc) + return StatusCode::FAILURE; + + auto idTable = m_podioLegacyDataSvc->getCollectionIDs(); + for (auto& name : m_collectionNames) { + debug() << "Finding collection " << name << " in collection registry." << endmsg; + if (!idTable->present(name)) { + error() << "Requested product " << name << " not found." << endmsg; + return StatusCode::FAILURE; + } + m_collectionIDs.push_back(idTable->collectionID(name)); + } + return StatusCode::SUCCESS; +} + +StatusCode PodioLegacyInput::execute() { + size_t cntr = 0; + // Re-create the collections from ROOT file + for (auto& id : m_collectionIDs) { + const std::string& collName = m_collectionNames.value().at(cntr++); + debug() << "Registering collection to read " << collName << " with id " << id << endmsg; + if (m_podioLegacyDataSvc->readCollection(collName, id).isFailure()) { + return StatusCode::FAILURE; + } + } + // Tell data service that we are done with requested collections + m_podioLegacyDataSvc->endOfRead(); + return StatusCode::SUCCESS; +} + +StatusCode PodioLegacyInput::finalize() { + if (GaudiAlgorithm::finalize().isFailure()) + return StatusCode::FAILURE; + return StatusCode::SUCCESS; +} diff --git a/k4FWCore/components/PodioLegacyInput.h b/k4FWCore/components/PodioLegacyInput.h new file mode 100644 index 00000000..58907418 --- /dev/null +++ b/k4FWCore/components/PodioLegacyInput.h @@ -0,0 +1,41 @@ +#ifndef FWCORE_PODIOLEGACYINPUT_H +#define FWCORE_PODIOLEGACYINPUT_H +// Gaaudi +#include "GaudiAlg/GaudiAlgorithm.h" + +// STL +#include +#include + +// forward declarations +// from k4FWCore: +class PodioLegacyDataSvc; + +/** @class PodioLegacyInput + * + * Class that allows to read ROOT files written with PodioOutput + * + * @author J. Lingemann + */ + +class PodioLegacyInput : public GaudiAlgorithm { +public: + /// Constructor. + PodioLegacyInput(const std::string& name, ISvcLocator* svcLoc); + /// Initialization of PodioInput. Acquires the data service, opens root file and creates trees. + virtual StatusCode initialize(); + /// Execute. Re-creates collections that are specified to be read and sets references. + virtual StatusCode execute(); + /// Finalize. Closes ROOT file. + virtual StatusCode finalize(); + +private: + /// Name of collections to read. Set by option collections (this is temporary) + Gaudi::Property> m_collectionNames{this, "collections", {}, "Places of collections to read"}; + /// Collection IDs (retrieved with CollectionIDTable from ROOT file, using collection names) + std::vector m_collectionIDs; + /// Data service: needed to register objects and get collection IDs. Just an observing pointer. + PodioLegacyDataSvc* m_podioLegacyDataSvc; +}; + +#endif diff --git a/k4FWCore/components/PodioLegacyOutput.cpp b/k4FWCore/components/PodioLegacyOutput.cpp new file mode 100644 index 00000000..72a11ebc --- /dev/null +++ b/k4FWCore/components/PodioLegacyOutput.cpp @@ -0,0 +1,200 @@ +#include + +#include "PodioLegacyOutput.h" +#include "TFile.h" +#include "k4FWCore/PodioLegacyDataSvc.h" +#include "rootUtils.h" + +DECLARE_COMPONENT(PodioLegacyOutput) + +PodioLegacyOutput::PodioLegacyOutput(const std::string& name, ISvcLocator* svcLoc) + : GaudiAlgorithm(name, svcLoc), m_firstEvent(true) {} + +StatusCode PodioLegacyOutput::initialize() { + if (GaudiAlgorithm::initialize().isFailure()) + return StatusCode::FAILURE; + + // check whether we have the PodioEvtSvc active + m_podioLegacyDataSvc = dynamic_cast(evtSvc().get()); + if (nullptr == m_podioLegacyDataSvc) { + error() << "Could not get DataSvc!" << endmsg; + return StatusCode::FAILURE; + } + + m_file = std::unique_ptr(TFile::Open(m_filename.value().c_str(), "RECREATE", "data file")); + // Both trees are written to the ROOT file and owned by it + // PodioDataSvc has ownership of EventDataTree + m_datatree = m_podioLegacyDataSvc->eventDataTree(); + m_datatree->SetDirectory(m_file.get()); + m_metadatatree = new TTree("metadata", "Metadata tree"); + m_runMDtree = new TTree("run_metadata", "Run metadata tree"); + m_evtMDtree = new TTree("evt_metadata", "Event metadata tree"); + m_colMDtree = new TTree("col_metadata", "Collection metadata tree"); + + m_evtMDtree->Branch("evtMD", "GenericParameters", m_podioLegacyDataSvc->getProvider().eventMetaDataPtr()); + m_switch = KeepDropSwitch(m_outputCommands); + return StatusCode::SUCCESS; +} + +void PodioLegacyOutput::resetBranches(const std::vector>& collections) { + for (auto& collNamePair : collections) { + auto collName = collNamePair.first; + if (m_switch.isOn(collName)) { + // Reconnect branches and collections + const auto collBuffers = collNamePair.second->getBuffers(); + m_datatree->SetBranchAddress(collName.c_str(), collBuffers.data); + auto colls = collBuffers.references; + if (colls != nullptr) { + for (size_t j = 0; j < colls->size(); ++j) { + auto l_branch = m_datatree->GetBranch((collName + "#" + std::to_string(j)).c_str()); + l_branch->SetAddress(&(*colls)[j]); + } + } + auto colls_v = collBuffers.vectorMembers; + if (colls_v != nullptr) { + int j = 0; + for (auto& c : (*colls_v)) { + void* add = c.second; + m_datatree->SetBranchAddress((collName + "_" + std::to_string(j)).c_str(), add); + ++j; + } + } + } + collNamePair.second->prepareForWrite(); + } +} + +void PodioLegacyOutput::createBranches(const std::vector>& collections) { + // collectionID, collection type, subset collection + std::vector>* collectionInfo = + new std::vector>(); + collectionInfo->reserve(collections.size()); + + for (auto& collNamePair : collections) { + auto collName = collNamePair.first; + std::string className(collNamePair.second->getValueTypeName()); + std::string collClassName = "vector<" + className + "Data>"; + int isOn = 0; + if (m_switch.isOn(collName)) { + isOn = 1; + const auto collBuffers = collNamePair.second->getBuffers(); + m_datatree->Branch(collName.c_str(), collClassName.c_str(), collBuffers.data); + // Create branches for collections holding relations + if (auto refColls = collBuffers.references) { + int i = 0; + for (auto& c : (*refColls)) { + const auto brName = podio::root_utils::refBranch(collName, i); + m_datatree->Branch(brName.c_str(), c.get()); + ++i; + } + // ---- vector members + auto vminfo = collBuffers.vectorMembers; + if (vminfo != nullptr) { + int i = 0; + for (auto& c : (*vminfo)) { + std::string typeName = "vector<" + c.first + ">"; + void* add = c.second; + m_datatree->Branch((collName + "_" + std::to_string(i)).c_str(), typeName.c_str(), add); + ++i; + } + } + } + } + + const auto collID = m_podioLegacyDataSvc->getCollectionIDs()->collectionID(collName); + // No check necessary, only registered collections possible + auto coll = collNamePair.second; + const auto collType = coll->getValueTypeName() + "Collection"; + collectionInfo->emplace_back(collID, std::move(collType), coll->isSubsetCollection()); + //} + + debug() << isOn << " Registering collection " << collClassName << " " << collName.c_str() << " containing type " + << className << endmsg; + collNamePair.second->prepareForWrite(); + } + + m_metadatatree->Branch("CollectionTypeInfo", collectionInfo); +} + +StatusCode PodioLegacyOutput::execute() { + // for now assume identical content for every event + // register for writing + if (m_firstEvent) { + createBranches(m_podioLegacyDataSvc->getCollections()); + } else { + resetBranches(m_podioLegacyDataSvc->getCollections()); + } + m_firstEvent = false; + debug() << "Filling DataTree .." << endmsg; + m_datatree->Fill(); + m_evtMDtree->Fill(); + return StatusCode::SUCCESS; +} + +/** PodioLegacyOutput::finalize +* has to happen after all algorithms that touch the data store finish. +* Here the job options are retrieved and stored to disk as a branch +* in the metadata tree. +* +*/ +StatusCode PodioLegacyOutput::finalize() { + if (GaudiAlgorithm::finalize().isFailure()) + return StatusCode::FAILURE; + //// prepare job options metadata /////////////////////// + // retrieve the configuration of the job + // and write it to file as vector of strings + std::vector config_data; + const auto& jobOptionsSvc = Gaudi::svcLocator()->getOptsSvc(); + const auto& configured_properties = jobOptionsSvc.items(); + for (const auto& per_property : configured_properties) { + std::stringstream config_stream; + // sample output: + // HepMCToEDMConverter.genparticles = "GenParticles"; + // Note that quotes are added to all property values, + // which leads to problems with ints, lists, dicts and bools. + // For theses types, the quotes must be removed in postprocessing. + config_stream << std::get<0>(per_property) << " = \"" << std::get<1>(per_property) << "\";" << std::endl; + config_data.push_back(config_stream.str()); + } + // Some default components are not captured by the job option service + // and have to be traversed like this. Note that Gaudi!577 will improve this. + for (const auto* name : {"ApplicationMgr", "MessageSvc", "NTupleSvc"}) { + std::stringstream config_stream; + auto svc = service(name); + if (!svc.isValid()) + continue; + for (const auto* property : svc->getProperties()) { + config_stream << name << "." << property->name() << " = \"" << property->toString() << "\";" << std::endl; + } + config_data.push_back(config_stream.str()); + } + //// finalize trees and file ////////////////////////////// + m_file->cd(); + + if (const char* env_key4hep_stack = std::getenv("KEY4HEP_STACK")) { + std::string s_env_key4hep_stack = env_key4hep_stack; + m_metadatatree->Branch("key4hepStack", &s_env_key4hep_stack); + } + + m_metadatatree->Branch("gaudiConfigOptions", &config_data); + m_metadatatree->Branch("CollectionIDs", m_podioLegacyDataSvc->getCollectionIDs()); + + m_metadatatree->Fill(); + + m_colMDtree->Branch("colMD", "std::map", + m_podioLegacyDataSvc->getProvider().getColMetaDataMap()); + m_colMDtree->Fill(); + m_runMDtree->Branch("runMD", "std::map", + m_podioLegacyDataSvc->getProvider().getRunMetaDataMap()); + m_runMDtree->Fill(); + + m_datatree->Write(); + m_file->Write(); + m_file->Close(); + info() << "Data written to: " << m_filename.value(); + if (!m_filenameRemote.value().empty()) { + TFile::Cp(m_filename.value().c_str(), m_filenameRemote.value().c_str(), false); + info() << " and copied to: " << m_filenameRemote.value() << endmsg; + } + return StatusCode::SUCCESS; +} diff --git a/k4FWCore/components/PodioLegacyOutput.h b/k4FWCore/components/PodioLegacyOutput.h new file mode 100644 index 00000000..80173dda --- /dev/null +++ b/k4FWCore/components/PodioLegacyOutput.h @@ -0,0 +1,59 @@ +#ifndef FWCORE_PODIOLEGACYOUTPUT_H +#define FWCORE_PODIOLEGACYOUTPUT_H + +#include "GaudiAlg/GaudiAlgorithm.h" +#include "k4FWCore/KeepDropSwitch.h" +#include "podio/CollectionBase.h" + +#include "TTree.h" + +#include + +// forward declarations +class TFile; +class PodioLegacyDataSvc; + +class PodioLegacyOutput : public GaudiAlgorithm { +public: + /// Constructor. + PodioLegacyOutput(const std::string& name, ISvcLocator* svcLoc); + + /// Initialization of PodioOutput. Acquires the data service, creates trees and root file. + virtual StatusCode initialize(); + /// Execute. For the first event creates branches for all collections known to PodioDataSvc and prepares them for + /// writing. For the following events it reconnects the branches with collections and prepares them for write. + virtual StatusCode execute(); + /// Finalize. Writes the meta data tree; writes file and cleans up all ROOT-pointers. + virtual StatusCode finalize(); + +private: + void resetBranches(const std::vector>& collections); + void createBranches(const std::vector>& collections); + /// First event or not + bool m_firstEvent; + /// Root file name the output is written to + Gaudi::Property m_filename{this, "filename", "output.root", "Name of the file to create"}; + /// Commands which output is to be kept + Gaudi::Property> m_outputCommands{ + this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."}; + Gaudi::Property m_filenameRemote{this, "remoteFilename", "", + "An optional file path to copy the outputfile to."}; + /// Switch for keeping or dropping outputs + KeepDropSwitch m_switch; + /// Needed for collection ID table + PodioLegacyDataSvc* m_podioLegacyDataSvc; + /// The actual ROOT file + std::unique_ptr m_file; + /// The tree to be filled with collections + TTree* m_datatree; + /// The tree to be filled with meta data + TTree* m_metadatatree; + TTree* m_runMDtree; + TTree* m_evtMDtree; + TTree* m_colMDtree; + + /// The stored collections + std::vector m_storedCollections; +}; + +#endif diff --git a/k4FWCore/components/PodioOutput.cpp b/k4FWCore/components/PodioOutput.cpp index ecde6e27..0bf718bc 100644 --- a/k4FWCore/components/PodioOutput.cpp +++ b/k4FWCore/components/PodioOutput.cpp @@ -21,113 +21,31 @@ StatusCode PodioOutput::initialize() { return StatusCode::FAILURE; } - m_file = std::unique_ptr(TFile::Open(m_filename.value().c_str(), "RECREATE", "data file")); - // Both trees are written to the ROOT file and owned by it - // PodioDataSvc has ownership of EventDataTree - m_datatree = m_podioDataSvc->eventDataTree(); - m_datatree->SetDirectory(m_file.get()); - m_metadatatree = new TTree("metadata", "Metadata tree"); - m_runMDtree = new TTree("run_metadata", "Run metadata tree"); - m_evtMDtree = new TTree("evt_metadata", "Event metadata tree"); - m_colMDtree = new TTree("col_metadata", "Collection metadata tree"); - - m_evtMDtree->Branch("evtMD", "GenericParameters", m_podioDataSvc->getProvider().eventMetaDataPtr()); + m_framewriter = std::make_unique(m_filename); m_switch = KeepDropSwitch(m_outputCommands); - return StatusCode::SUCCESS; -} - -void PodioOutput::resetBranches(const std::vector>& collections) { - for (auto& collNamePair : collections) { - auto collName = collNamePair.first; - if (m_switch.isOn(collName)) { - // Reconnect branches and collections - const auto collBuffers = collNamePair.second->getBuffers(); - m_datatree->SetBranchAddress(collName.c_str(), collBuffers.data); - auto colls = collBuffers.references; - if (colls != nullptr) { - for (size_t j = 0; j < colls->size(); ++j) { - auto l_branch = m_datatree->GetBranch((collName + "#" + std::to_string(j)).c_str()); - l_branch->SetAddress(&(*colls)[j]); - } - } - auto colls_v = collBuffers.vectorMembers; - if (colls_v != nullptr) { - int j = 0; - for (auto& c : (*colls_v)) { - void* add = c.second; - m_datatree->SetBranchAddress((collName + "_" + std::to_string(j)).c_str(), add); - ++j; - } - } - } - collNamePair.second->prepareForWrite(); - } -} -void PodioOutput::createBranches(const std::vector>& collections) { - // collectionID, collection type, subset collection - std::vector>* collectionInfo = - new std::vector>(); - collectionInfo->reserve(collections.size()); - - for (auto& collNamePair : collections) { - auto collName = collNamePair.first; - std::string className(collNamePair.second->getValueTypeName()); - std::string collClassName = "vector<" + className + "Data>"; - int isOn = 0; - if (m_switch.isOn(collName)) { - isOn = 1; - const auto collBuffers = collNamePair.second->getBuffers(); - m_datatree->Branch(collName.c_str(), collClassName.c_str(), collBuffers.data); - // Create branches for collections holding relations - if (auto refColls = collBuffers.references) { - int i = 0; - for (auto& c : (*refColls)) { - const auto brName = podio::root_utils::refBranch(collName, i); - m_datatree->Branch(brName.c_str(), c.get()); - ++i; - } - // ---- vector members - auto vminfo = collBuffers.vectorMembers; - if (vminfo != nullptr) { - int i = 0; - for (auto& c : (*vminfo)) { - std::string typeName = "vector<" + c.first + ">"; - void* add = c.second; - m_datatree->Branch((collName + "_" + std::to_string(i)).c_str(), typeName.c_str(), add); - ++i; - } - } - } - } - - const auto collID = m_podioDataSvc->getCollectionIDs()->collectionID(collName); - // No check necessary, only registered collections possible - auto coll = collNamePair.second; - const auto collType = coll->getValueTypeName() + "Collection"; - collectionInfo->emplace_back(collID, std::move(collType), coll->isSubsetCollection()); - //} - - debug() << isOn << " Registering collection " << collClassName << " " << collName.c_str() << " containing type " - << className << endmsg; - collNamePair.second->prepareForWrite(); - } - - m_metadatatree->Branch("CollectionTypeInfo", collectionInfo); + return StatusCode::SUCCESS; } StatusCode PodioOutput::execute() { + auto& frame = m_podioDataSvc->getFrame(); + // for now assume identical content for every event // register for writing if (m_firstEvent) { - createBranches(m_podioDataSvc->getCollections()); + auto collections = frame.getAvailableCollections(); + std::vector collection_names_to_write; + for (auto collection_name : collections) { + if (m_switch.isOn(collection_name)) { + collection_names_to_write.push_back(collection_name); + } + } + m_framewriter->writeFrame(frame, "events", collection_names_to_write); } else { - resetBranches(m_podioDataSvc->getCollections()); + m_framewriter->writeFrame(frame, "events"); } m_firstEvent = false; - debug() << "Filling DataTree .." << endmsg; - m_datatree->Fill(); - m_evtMDtree->Fill(); + return StatusCode::SUCCESS; } @@ -168,33 +86,20 @@ StatusCode PodioOutput::finalize() { } config_data.push_back(config_stream.str()); } - //// finalize trees and file ////////////////////////////// - m_file->cd(); + // Collect all the metadata + podio::Frame metadata_frame{}; + metadata_frame.putParameter("gaudiConfigOptions", config_data); if (const char* env_key4hep_stack = std::getenv("KEY4HEP_STACK")) { std::string s_env_key4hep_stack = env_key4hep_stack; - m_metadatatree->Branch("key4hepStack", &s_env_key4hep_stack); + metadata_frame.putParameter("key4hepstack", s_env_key4hep_stack); } + m_framewriter->writeFrame(metadata_frame, "metadata"); - m_metadatatree->Branch("gaudiConfigOptions", &config_data); - m_metadatatree->Branch("CollectionIDs", m_podioDataSvc->getCollectionIDs()); + // TODO: Create frames for run metadata and collection metadata - m_metadatatree->Fill(); + // write information into file + m_framewriter->finish(); - m_colMDtree->Branch("colMD", "std::map", - m_podioDataSvc->getProvider().getColMetaDataMap()); - m_colMDtree->Fill(); - m_runMDtree->Branch("runMD", "std::map", - m_podioDataSvc->getProvider().getRunMetaDataMap()); - m_runMDtree->Fill(); - - m_datatree->Write(); - m_file->Write(); - m_file->Close(); - info() << "Data written to: " << m_filename.value(); - if (!m_filenameRemote.value().empty()) { - TFile::Cp(m_filename.value().c_str(), m_filenameRemote.value().c_str(), false); - info() << " and copied to: " << m_filenameRemote.value() << endmsg; - } return StatusCode::SUCCESS; } diff --git a/k4FWCore/components/PodioOutput.h b/k4FWCore/components/PodioOutput.h index 80cb00c5..7ca65715 100644 --- a/k4FWCore/components/PodioOutput.h +++ b/k4FWCore/components/PodioOutput.h @@ -4,8 +4,7 @@ #include "GaudiAlg/GaudiAlgorithm.h" #include "k4FWCore/KeepDropSwitch.h" #include "podio/CollectionBase.h" - -#include "TTree.h" +#include "podio/ROOTFrameWriter.h" #include @@ -27,8 +26,6 @@ class PodioOutput : public GaudiAlgorithm { virtual StatusCode finalize(); private: - void resetBranches(const std::vector>& collections); - void createBranches(const std::vector>& collections); /// First event or not bool m_firstEvent; /// Root file name the output is written to @@ -42,16 +39,8 @@ class PodioOutput : public GaudiAlgorithm { KeepDropSwitch m_switch; /// Needed for collection ID table PodioDataSvc* m_podioDataSvc; - /// The actual ROOT file - std::unique_ptr m_file; - /// The tree to be filled with collections - TTree* m_datatree; - /// The tree to be filled with meta data - TTree* m_metadatatree; - TTree* m_runMDtree; - TTree* m_evtMDtree; - TTree* m_colMDtree; - + /// The actual ROOT frame writer + std::unique_ptr m_framewriter; /// The stored collections std::vector m_storedCollections; }; diff --git a/k4FWCore/components/k4LegacyDataSvc.cpp b/k4FWCore/components/k4LegacyDataSvc.cpp new file mode 100644 index 00000000..8c33416b --- /dev/null +++ b/k4FWCore/components/k4LegacyDataSvc.cpp @@ -0,0 +1,16 @@ + +#include "k4LegacyDataSvc.h" + +// Instantiation of a static factory class used by clients to create +// instances of this service +DECLARE_COMPONENT(k4LegacyDataSvc) + +/// Standard Constructor +k4LegacyDataSvc::k4LegacyDataSvc(const std::string& name, ISvcLocator* svc) : PodioLegacyDataSvc(name, svc) { + declareProperty("inputs", m_filenames = {}, "Names of the files to read"); + declareProperty("input", m_filename = "", "Name of the file to read"); + declareProperty("FirstEventEntry", m_1stEvtEntry = 0, "First event to read"); +} + +/// Standard Destructor +k4LegacyDataSvc::~k4LegacyDataSvc() {} diff --git a/k4FWCore/components/k4LegacyDataSvc.h b/k4FWCore/components/k4LegacyDataSvc.h new file mode 100644 index 00000000..63018001 --- /dev/null +++ b/k4FWCore/components/k4LegacyDataSvc.h @@ -0,0 +1,15 @@ + +#ifndef K4FWCORE_K4LEGACYDATASVC_H +#define K4FWCORE_K4LEGACYDATASVC_H + +#include "k4FWCore/PodioLegacyDataSvc.h" + +class k4LegacyDataSvc : public PodioLegacyDataSvc { +public: + /// Standard Constructor + k4LegacyDataSvc(const std::string& name, ISvcLocator* svc); + + /// Standard Destructor + virtual ~k4LegacyDataSvc(); +}; +#endif diff --git a/k4FWCore/components/k4TestFloatProducer.cpp b/k4FWCore/components/k4TestFloatProducer.cpp deleted file mode 100644 index 7fa0fa78..00000000 --- a/k4FWCore/components/k4TestFloatProducer.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "k4TestFloatProducer.h" - -DECLARE_COMPONENT(k4TestFloatProducer) - -k4TestFloatProducer::k4TestFloatProducer(const std::string& name, ISvcLocator* svcLoc) : GaudiAlgorithm(name, svcLoc) { - declareProperty("Float", m_datahandle_float, "Float Value handle (output)"); -} - -// setup before event processing -// (standard boilerplate code) -StatusCode k4TestFloatProducer::initialize() { - if (GaudiAlgorithm::initialize().isFailure()) - return StatusCode::FAILURE; - return StatusCode::SUCCESS; -} - -// process events -// create a single float as output -// value is taken from options. -StatusCode k4TestFloatProducer::execute() { - // the datahandle takes ownership of the memory, - // need to pass a raw pointer. - float* _float = new float(); - *_float = m_floatValue.value(); - m_datahandle_float.put(_float); - return StatusCode::SUCCESS; -} - -// after all events have been processed: -// (standard boilerplate code) -StatusCode k4TestFloatProducer::finalize() { - if (GaudiAlgorithm::finalize().isFailure()) - return StatusCode::FAILURE; - return StatusCode::SUCCESS; -} diff --git a/k4FWCore/components/k4TestFloatProducer.h b/k4FWCore/components/k4TestFloatProducer.h deleted file mode 100644 index 09c44a98..00000000 --- a/k4FWCore/components/k4TestFloatProducer.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef K4FWCORE_TESTFLOATPRODUCER_H -#define K4FWCORE_TESTFLOATPRODUCER_H - -#include "GaudiAlg/GaudiAlgorithm.h" -#include "k4FWCore/DataHandle.h" - -/*** - * Simple test algorithm which outputs a single float each event - * The value of the float can be set in the options, - * using the FloatValue property. - * - */ -class k4TestFloatProducer : public GaudiAlgorithm { -public: - k4TestFloatProducer(const std::string& name, ISvcLocator* svcLoc); - - virtual StatusCode initialize(); - virtual StatusCode execute(); - virtual StatusCode finalize(); - -private: - /// value which can be set from options - Gaudi::Property m_floatValue{this, "FloatValue", 0.0, "Value of Float to produce"}; - /// datahandle for the output - DataHandle m_datahandle_float{"Float", Gaudi::DataHandle::Writer, this}; -}; - -#endif diff --git a/k4FWCore/include/k4FWCore/DataHandle.h b/k4FWCore/include/k4FWCore/DataHandle.h index 43404884..3499d06d 100644 --- a/k4FWCore/include/k4FWCore/DataHandle.h +++ b/k4FWCore/include/k4FWCore/DataHandle.h @@ -3,6 +3,7 @@ #include "k4FWCore/DataWrapper.h" #include "k4FWCore/PodioDataSvc.h" +#include "k4FWCore/PodioLegacyDataSvc.h" #include "Gaudi/Algorithm.h" #include "GaudiKernel/DataObjectHandle.h" @@ -39,9 +40,6 @@ template class DataHandle : public DataObjectHandle> T* put(std::unique_ptr object); - ///Retrieve CellID string from the Collection metadata - const std::string getCollMetadataCellID(const unsigned int id); - /** * Create and register object in transient store */ @@ -71,10 +69,21 @@ DataHandle::DataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode : DataObjectHandle>(descriptor, a, fatherAlg), m_eds("EventDataSvc", "DataHandle") { if (a == Gaudi::DataHandle::Writer) { StatusCode sc = m_eds.retrieve(); - PodioDataSvc* pds; - pds = dynamic_cast(m_eds.get()); m_dataPtr = nullptr; - if (nullptr != pds) { + PodioDataSvc* podio_data_service; + + podio_data_service = dynamic_cast(m_eds.get()); + if (nullptr != podio_data_service) { + if (!std::is_convertible::value) { + throw GaudiException("DataHandle", "Trying to add non-PODIO type to data service", StatusCode::FAILURE); + + } else { m_dataPtr = new T();} + } else { + + // This is the legacy implementation kept for a transition period + PodioLegacyDataSvc* plds; + plds = dynamic_cast(m_eds.get()); + if (nullptr != plds) { if (std::is_convertible::value) { // case 1: T is a podio collection // for this case creation of branches is still handled in PodioOutput @@ -86,13 +95,13 @@ DataHandle::DataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode // and have to append a char indicating type (see TTree documentation) // therefore space needs to be allocated for the integer m_dataPtr = new T(); - TTree* tree = pds->eventDataTree(); + TTree* tree = plds->eventDataTree(); tree->Branch(descriptor.c_str(), m_dataPtr, (descriptor + "/I").c_str()); } else if constexpr (std::is_floating_point_v) { // case 3: T is some floating point type // similar case 2, distinguish floats and doubles by size m_dataPtr = new T(); - TTree* tree = pds->eventDataTree(); + TTree* tree = plds->eventDataTree(); if (sizeof(T) > 4) { tree->Branch(descriptor.c_str(), m_dataPtr, (descriptor + "/D").c_str()); } else { @@ -102,11 +111,12 @@ DataHandle::DataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode // case 4: T is any other type (for which exists a root dictionary, // otherwise i/o will fail) // this includes std::vectors of ints, floats - TTree* tree = pds->eventDataTree(); + TTree* tree = plds->eventDataTree(); tree->Branch(descriptor.c_str(), &m_dataPtr); } } } + } } /** @@ -168,23 +178,6 @@ template T* DataHandle::put(std::unique_ptr objectp) { return objectp.release(); } -//--------------------------------------------------------------------------- -// Get the CellID encoded String from the Metadata of the collection -// using Podio Data Service -// Give collection ID -template const std::string DataHandle::getCollMetadataCellID(const unsigned int id) { - PodioDataSvc* pds; - pds = dynamic_cast(m_eds.get()); - - if (pds != nullptr) { - auto colMD = pds->getProvider().getCollectionMetaData(id); - return colMD.getValue("CellIDEncodingString"); - } else { - std::string msg("Could not get Podio Data Service."); - throw GaudiException(msg, "Failed to get Collection Metadata.", StatusCode::FAILURE); - } -} - //--------------------------------------------------------------------------- /** * Create the collection, put it in the DataObjectHandle and return the diff --git a/k4FWCore/include/k4FWCore/DataWrapper.h b/k4FWCore/include/k4FWCore/DataWrapper.h index 182a85d4..289da335 100644 --- a/k4FWCore/include/k4FWCore/DataWrapper.h +++ b/k4FWCore/include/k4FWCore/DataWrapper.h @@ -13,6 +13,7 @@ class GAUDI_API DataWrapperBase : public DataObject { // DataSvc would need a templated register method virtual podio::CollectionBase* collectionBase() = 0; virtual ~DataWrapperBase(){}; + virtual void resetData() = 0; }; template class GAUDI_API DataWrapper : public DataWrapperBase { @@ -21,12 +22,13 @@ template class GAUDI_API DataWrapper : public DataWrapperBase { virtual ~DataWrapper(); const T* getData() { return m_data; } - void setData(T* data) { m_data = data; } + void setData(const T* data) { m_data = data; } + virtual void resetData() { m_data = nullptr; } /// try to cast to collectionBase; may return nullptr; virtual podio::CollectionBase* collectionBase(); private: - T* m_data; + const T* m_data; }; template DataWrapper::~DataWrapper() { @@ -36,7 +38,7 @@ template DataWrapper::~DataWrapper() { template podio::CollectionBase* DataWrapper::collectionBase() { if (std::is_base_of::value) { - return reinterpret_cast(m_data); + return reinterpret_cast(const_cast(m_data)); } return nullptr; } diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index 01c33284..db27c03c 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -6,11 +6,11 @@ // PODIO #include "podio/CollectionBase.h" #include "podio/CollectionIDTable.h" -#include "podio/EventStore.h" -#include "podio/ROOTReader.h" - +#include "podio/ROOTFrameReader.h" +#include "podio/Frame.h" #include // Forward declarations +class DataWrapperBase; /** @class PodioEvtSvc EvtDataSvc.h * @@ -40,36 +40,29 @@ class PodioDataSvc : public DataSvc { virtual StatusCode registerObject(std::string_view parentPath, std::string_view fullPath, DataObject* pObject) override final; - StatusCode readCollection(const std::string& collectionName, int collectionID); + StatusCode readCollection(const std::string& collectionName); - virtual const CollRegistry& getCollections() const { return m_collections; } - podio::EventStore& getProvider() { return m_provider; } - virtual podio::CollectionIDTable* getCollectionIDs() { return m_collectionIDs; } + const podio::Frame& getFrame() const { return m_frame; } - /// Set the collection IDs (if reading a file) - void setCollectionIDs(podio::CollectionIDTable* collectionIds); /// Resets caches of reader and event store, increases event counter void endOfRead(); - TTree* eventDataTree() { return m_eventDataTree; } - private: - // eventDataTree - TTree* m_eventDataTree; /// PODIO reader for ROOT files - podio::ROOTReader m_reader; - /// PODIO EventStore, used to initialise collections - podio::EventStore m_provider; + podio::ROOTFrameReader m_reader; + /// PODIO Frame, used to initialise collections + podio::Frame m_frame; /// Counter of the event number int m_eventNum{0}; /// Number of events in the file / to process int m_eventMax{-1}; + /// Whether reading from file at all + bool m_reading_from_file{false}; SmartIF m_cnvSvc; - // special members for podio handling - std::vector> m_collections; - podio::CollectionIDTable* m_collectionIDs; + // Registry of data wrappers; needed for memory management + std::vector m_podio_datawrappers; protected: /// ROOT file name the input is read from. Set by option filename diff --git a/k4FWCore/include/k4FWCore/PodioLegacyDataSvc.h b/k4FWCore/include/k4FWCore/PodioLegacyDataSvc.h new file mode 100644 index 00000000..6b2f6f79 --- /dev/null +++ b/k4FWCore/include/k4FWCore/PodioLegacyDataSvc.h @@ -0,0 +1,82 @@ +#ifndef FWCORE_PODIOLEGACYDATASVC_H +#define FWCORE_PODIOLEGACYDATASVC_H + +#include "GaudiKernel/DataSvc.h" +#include "GaudiKernel/IConversionSvc.h" +// PODIO +#include "podio/CollectionBase.h" +#include "podio/CollectionIDTable.h" +#include "podio/EventStore.h" +#include "podio/ROOTReader.h" + +#include +// Forward declarations + +/** @class PodioLegacyEvtSvc EvtDataSvc.h + * + * An EvtDataSvc for PODIO classes + * + * @author B. Hegner + */ +class PodioLegacyDataSvc : public DataSvc { +public: + typedef std::vector> CollRegistry; + + virtual StatusCode initialize(); + virtual StatusCode reinitialize(); + virtual StatusCode finalize(); + virtual StatusCode clearStore(); + + /// Standard Constructor + PodioLegacyDataSvc(const std::string& name, ISvcLocator* svc); + + /// Standard Destructor + virtual ~PodioLegacyDataSvc(); + + // Use DataSvc functionality except where we override + using DataSvc::registerObject; + /// Overriding standard behaviour of evt service + /// Register object with the data store. + virtual StatusCode registerObject(std::string_view parentPath, std::string_view fullPath, + DataObject* pObject) override final; + + StatusCode readCollection(const std::string& collectionName, int collectionID); + + virtual const CollRegistry& getCollections() const { return m_collections; } + podio::EventStore& getProvider() { return m_provider; } + virtual podio::CollectionIDTable* getCollectionIDs() { return m_collectionIDs; } + + /// Set the collection IDs (if reading a file) + void setCollectionIDs(podio::CollectionIDTable* collectionIds); + /// Resets caches of reader and event store, increases event counter + void endOfRead(); + + TTree* eventDataTree() { return m_eventDataTree; } + +private: + // eventDataTree + TTree* m_eventDataTree; + /// PODIO reader for ROOT files + podio::ROOTReader m_reader; + /// PODIO EventStore, used to initialise collections + podio::EventStore m_provider; + /// Counter of the event number + int m_eventNum{0}; + /// Number of events in the file / to process + int m_eventMax{-1}; + + SmartIF m_cnvSvc; + + // special members for podio handling + std::vector> m_collections; + podio::CollectionIDTable* m_collectionIDs; + +protected: + /// ROOT file name the input is read from. Set by option filename + std::vector m_filenames; + std::string m_filename; + /// Jump to nth events at the beginning. Set by option FirstEventEntry + /// This option is helpful when we want to debug an event in the middle of a file + unsigned m_1stEvtEntry{0}; +}; +#endif // CORE_PODIOLEGACYDATASVC_H diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index 3f4408da..b20d29e5 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -23,20 +23,25 @@ StatusCode PodioDataSvc::initialize() { if (m_filenames.size() > 0) { if (m_filenames[0] != "") { + m_reading_from_file = true; m_reader.openFiles(m_filenames); - m_eventMax = m_reader.getEntries(); - - m_provider.setReader(&m_reader); - - auto idTable = m_provider.getCollectionIDTable(); - setCollectionIDs(idTable); + m_eventMax = m_reader.getEntries("events"); if (m_1stEvtEntry != 0) { - m_reader.goToEvent(m_1stEvtEntry); m_eventMax -= m_1stEvtEntry; } + + m_frame = podio::Frame(m_reader.readEntry("events",m_1stEvtEntry)); } + } else { + m_frame = podio::Frame(); + } + + // read + + // if empty, create a new frame from scratch + return status; } /// Service reinitialisation @@ -52,21 +57,27 @@ StatusCode PodioDataSvc::finalize() { } StatusCode PodioDataSvc::clearStore() { - for (auto& collNamePair : m_collections) { - if (collNamePair.second != nullptr) { - collNamePair.second->clear(); - } + // as the frame takes care of the ownership of the podio::Collections, + // make sure the DataWrappers don't cause a double delete + for(auto wrapper : m_podio_datawrappers){ + wrapper->resetData(); } + m_podio_datawrappers.clear(); + + // create a new frame + if (m_reading_from_file) { + m_frame = podio::Frame(m_reader.readNextEntry("events")); + } else { + m_frame = podio::Frame(); + } + DataSvc::clearStore().ignore(); - m_collections.clear(); return StatusCode::SUCCESS; } void PodioDataSvc::endOfRead() { StatusCode sc; if (m_eventMax != -1) { - m_provider.clearCaches(); - m_reader.endOfEvent(); if (m_eventNum++ > m_eventMax) { info() << "Reached end of file with event " << m_eventMax << endmsg; IEventProcessor* eventProcessor; @@ -77,36 +88,23 @@ void PodioDataSvc::endOfRead() { // todo: figure out sthg to do with sc (added to silence -Wunused-result) } -void PodioDataSvc::setCollectionIDs(podio::CollectionIDTable* collectionIds) { - if (m_collectionIDs != nullptr) { - delete m_collectionIDs; - } - m_collectionIDs = collectionIds; -} - /// Standard Constructor PodioDataSvc::PodioDataSvc(const std::string& name, ISvcLocator* svc) - : DataSvc(name, svc), m_collectionIDs(new podio::CollectionIDTable()) { - m_eventDataTree = new TTree("events", "Events tree"); + : DataSvc(name, svc) { } /// Standard Destructor PodioDataSvc::~PodioDataSvc() {} -StatusCode PodioDataSvc::readCollection(const std::string& collName, int collectionID) { - podio::CollectionBase* collection(nullptr); - - auto idTable = m_provider.getCollectionIDTable(); - m_provider.get(collectionID, collection); +StatusCode PodioDataSvc::readCollection(const std::string& collName) { + const podio::CollectionBase* collection(nullptr); + collection = m_frame.get(collName); if (collection->isSubsetCollection()) { return StatusCode::SUCCESS; } auto wrapper = new DataWrapper; - int id = m_collectionIDs->add(collName); - collection->setID(id); - collection->prepareAfterRead(); - wrapper->setData(collection); + wrapper->setData(collection); return registerObject("/Event", "/" + collName, wrapper); } @@ -117,9 +115,9 @@ StatusCode PodioDataSvc::registerObject(std::string_view parentPath, std::string if (coll != nullptr) { size_t pos = fullPath.find_last_of("/"); std::string shortPath(fullPath.substr(pos + 1, fullPath.length())); - int id = m_collectionIDs->add(shortPath); - coll->setID(id); - m_collections.emplace_back(std::make_pair(shortPath, coll)); + // Attention: this passes the ownership of the data to the frame + m_frame.put(std::unique_ptr(coll), shortPath); + m_podio_datawrappers.push_back(wrapper); } } return DataSvc::registerObject(parentPath, fullPath, pObject); diff --git a/k4FWCore/src/PodioLegacyDataSvc.cpp b/k4FWCore/src/PodioLegacyDataSvc.cpp new file mode 100644 index 00000000..bc32c4eb --- /dev/null +++ b/k4FWCore/src/PodioLegacyDataSvc.cpp @@ -0,0 +1,126 @@ +#include "k4FWCore/PodioLegacyDataSvc.h" +#include "GaudiKernel/IConversionSvc.h" +#include "GaudiKernel/IEventProcessor.h" +#include "GaudiKernel/ISvcLocator.h" + +#include "k4FWCore/DataWrapper.h" + +#include "TTree.h" + +/// Service initialisation +StatusCode PodioLegacyDataSvc::initialize() { + // Nothing to do: just call base class initialisation + StatusCode status = DataSvc::initialize(); + ISvcLocator* svc_loc = serviceLocator(); + + // Attach data loader facility + m_cnvSvc = svc_loc->service("EventPersistencySvc"); + status = setDataLoader(m_cnvSvc); + + if (m_filename != "") { + m_filenames.push_back(m_filename); + } + + if (m_filenames.size() > 0) { + if (m_filenames[0] != "") { + m_reader.openFiles(m_filenames); + m_eventMax = m_reader.getEntries(); + + m_provider.setReader(&m_reader); + + auto idTable = m_provider.getCollectionIDTable(); + setCollectionIDs(idTable); + + if (m_1stEvtEntry != 0) { + m_reader.goToEvent(m_1stEvtEntry); + m_eventMax -= m_1stEvtEntry; + } + } + } + return status; +} +/// Service reinitialisation +StatusCode PodioLegacyDataSvc::reinitialize() { + // Do nothing for this service + return StatusCode::SUCCESS; +} +/// Service finalization +StatusCode PodioLegacyDataSvc::finalize() { + m_cnvSvc = 0; // release + DataSvc::finalize().ignore(); + return StatusCode::SUCCESS; +} + +StatusCode PodioLegacyDataSvc::clearStore() { + for (auto& collNamePair : m_collections) { + if (collNamePair.second != nullptr) { + collNamePair.second->clear(); + } + } + DataSvc::clearStore().ignore(); + m_collections.clear(); + return StatusCode::SUCCESS; +} + +void PodioLegacyDataSvc::endOfRead() { + StatusCode sc; + if (m_eventMax != -1) { + m_provider.clearCaches(); + m_reader.endOfEvent(); + if (m_eventNum++ > m_eventMax) { + info() << "Reached end of file with event " << m_eventMax << endmsg; + IEventProcessor* eventProcessor; + sc = service("ApplicationMgr", eventProcessor); + sc = eventProcessor->stopRun(); + } + } + // todo: figure out sthg to do with sc (added to silence -Wunused-result) +} + +void PodioLegacyDataSvc::setCollectionIDs(podio::CollectionIDTable* collectionIds) { + if (m_collectionIDs != nullptr) { + delete m_collectionIDs; + } + m_collectionIDs = collectionIds; +} + +/// Standard Constructor +PodioLegacyDataSvc::PodioLegacyDataSvc(const std::string& name, ISvcLocator* svc) + : DataSvc(name, svc), m_collectionIDs(new podio::CollectionIDTable()) { + m_eventDataTree = new TTree("events", "Events tree"); +} + +/// Standard Destructor +PodioLegacyDataSvc::~PodioLegacyDataSvc() {} + +StatusCode PodioLegacyDataSvc::readCollection(const std::string& collName, int collectionID) { + podio::CollectionBase* collection(nullptr); + + auto idTable = m_provider.getCollectionIDTable(); + m_provider.get(collectionID, collection); + + if (collection->isSubsetCollection()) { + return StatusCode::SUCCESS; + } + auto wrapper = new DataWrapper; + int id = m_collectionIDs->add(collName); + collection->setID(id); + collection->prepareAfterRead(); + wrapper->setData(collection); + return registerObject("/Event", "/" + collName, wrapper); +} + +StatusCode PodioLegacyDataSvc::registerObject(std::string_view parentPath, std::string_view fullPath, DataObject* pObject) { + DataWrapperBase* wrapper = dynamic_cast(pObject); + if (wrapper != nullptr) { + podio::CollectionBase* coll = wrapper->collectionBase(); + if (coll != nullptr) { + size_t pos = fullPath.find_last_of("/"); + std::string shortPath(fullPath.substr(pos + 1, fullPath.length())); + int id = m_collectionIDs->add(shortPath); + coll->setID(id); + m_collections.emplace_back(std::make_pair(shortPath, coll)); + } + } + return DataSvc::registerObject(parentPath, fullPath, pObject); +} diff --git a/k4FWCore/test/options/simple_write.py b/k4FWCore/test/options/simple_write.py deleted file mode 100644 index 12d31422..00000000 --- a/k4FWCore/test/options/simple_write.py +++ /dev/null @@ -1,23 +0,0 @@ - -from Gaudi.Configuration import * -from Configurables import ApplicationMgr - -app = ApplicationMgr() -app.EvtSel = "NONE" -app.EvtMax = 10; - -from Configurables import k4DataSvc -dataservice = k4DataSvc("EventDataSvc") -ApplicationMgr().ExtSvc += [dataservice] - -from Configurables import k4TestFloatProducer -fprod = k4TestFloatProducer() -fprod.FloatValue = 5.4321 -fprod.Float.Path = "floatvalue" -ApplicationMgr().TopAlg += [fprod] - -from Configurables import PodioOutput -out = PodioOutput() -out.filename = "output_fwtest1.root" -out.outputCommands = ["keep *"] -ApplicationMgr().TopAlg += [out] diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index d38d31b5..d2fd749b 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -107,3 +107,8 @@ add_test(NAME TestUniqueIDGenSvc WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND ${K4RUN} options/TestUniqueIDGenSvc.py) set_test_env(TestUniqueIDGenSvc) + +add_test(NAME CreateLegacyExampleEventData + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND ${K4RUN} options/createLegacyExampleEventData.py) +set_test_env(CreateLegacyExampleEventData) \ No newline at end of file diff --git a/test/k4FWCoreTest/options/TwoProducers.py b/test/k4FWCoreTest/options/TwoProducers.py index 82f0ae70..6e734549 100644 --- a/test/k4FWCoreTest/options/TwoProducers.py +++ b/test/k4FWCoreTest/options/TwoProducers.py @@ -24,8 +24,6 @@ producer2.trackhits.Path = "trackhits2" producer2.simtrackhits.Path = "simtrackhits2" producer2.tracks.Path = "tracks2" -producer2.singleint.Path = "singleint2" -producer2.singlefloat.Path = "singlefloat2" producer2.vectorfloat.Path = "vectorfloat2" ApplicationMgr().TopAlg += [producer2] diff --git a/test/k4FWCoreTest/options/createLegacyExampleEventData.py b/test/k4FWCoreTest/options/createLegacyExampleEventData.py new file mode 100644 index 00000000..fdfab0d2 --- /dev/null +++ b/test/k4FWCoreTest/options/createLegacyExampleEventData.py @@ -0,0 +1,23 @@ +from Gaudi.Configuration import * + +from Configurables import k4LegacyDataSvc +podioevent = k4LegacyDataSvc("EventDataSvc") + +from Configurables import k4FWCoreTest_CreateExampleEventData +producer = k4FWCoreTest_CreateExampleEventData() + +from Configurables import PodioLegacyOutput +out = PodioLegacyOutput("out") +out.filename = "output_k4test_legacyexampledata.root" +out.outputCommands = ["keep *"] + +from Configurables import ApplicationMgr +ApplicationMgr( TopAlg=[producer, out], + EvtSel="NONE", + EvtMax=100, + ExtSvc=[podioevent], + OutputLevel=INFO, + StopOnSignal=True, + ) + + diff --git a/test/k4FWCoreTest/options/readLegacyExampleEventData.py b/test/k4FWCoreTest/options/readLegacyExampleEventData.py new file mode 100644 index 00000000..022cb531 --- /dev/null +++ b/test/k4FWCoreTest/options/readLegacyExampleEventData.py @@ -0,0 +1,24 @@ +from Gaudi.Configuration import * + +from Configurables import k4LegacyDataSvc +podioevent = k4LegacyDataSvc("EventDataSvc") +podioevent.input = "output_k4test_legacyexampledata.root" + +from Configurables import PodioLegacyInput +inp = PodioLegacyInput() +inp.collections = ["MCParticles", "SimTrackerHits", "TrackerHits", "Tracks"] + +from Configurables import PodioLegacyOutput +oup = PodioLegacyOutput() +oup.filename = "output_k4test_legacyexampledata_2.root" +oup.outputCommands = ["drop MCParticles"] + +from Configurables import ApplicationMgr +ApplicationMgr( TopAlg=[inp, oup], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[podioevent], + OutputLevel=DEBUG, + ) + + diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.cpp index 130d016f..ca573dac 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.cpp @@ -30,12 +30,10 @@ StatusCode k4FWCoreTest_AlgorithmWithTFile::initialize() { } StatusCode k4FWCoreTest_AlgorithmWithTFile::execute() { - m_singleIntHandle.put(new int(12345)); - m_singleFloatHandle.put(new float(12345.6789)); - std::vector* floatVector = m_vectorFloatHandle.createAndPut(); - floatVector->emplace_back(125.); - floatVector->emplace_back(25.); + auto* floatVector = m_vectorFloatHandle.createAndPut(); + floatVector->push_back(125.); + floatVector->push_back(25.); edm4hep::MCParticleCollection* particles = m_mcParticleHandle.createAndPut(); diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.h index af59f9a5..0440e49f 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_AlgorithmWithTFile.h @@ -9,6 +9,8 @@ #include "TTree.h" #include "k4FWCore/DataHandle.h" +#include "podio/UserDataCollection.h" + // datamodel namespace edm4hep { class MCParticleCollection; @@ -47,9 +49,7 @@ class k4FWCoreTest_AlgorithmWithTFile : public GaudiAlgorithm { /// Handle for the genvertices to be written DataHandle m_simTrackerHitHandle{"SimTrackerHit", Gaudi::DataHandle::Writer, this}; - DataHandle m_singleFloatHandle{"SingleFloat", Gaudi::DataHandle::Writer, this}; - DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; - DataHandle m_singleIntHandle{"SingleInt", Gaudi::DataHandle::Writer, this}; + DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; /// for testing: write a second TFile by user in an algorithm Float_t m_value; diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp index 63c60c88..da742de7 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp @@ -1,5 +1,7 @@ #include "k4FWCoreTest_CreateExampleEventData.h" +#include "podio/UserDataCollection.h" + // datamodel #include "edm4hep/MCParticleCollection.h" #include "edm4hep/SimTrackerHitCollection.h" @@ -14,9 +16,7 @@ k4FWCoreTest_CreateExampleEventData::k4FWCoreTest_CreateExampleEventData(const s declareProperty("simtrackhits", m_simTrackerHitHandle, "Dummy Hit collection (output)"); declareProperty("trackhits", m_TrackerHitHandle, "Dummy Hit collection (output)"); declareProperty("tracks", m_trackHandle, "Dummy track collection (output)"); - declareProperty("singlefloat", m_singleFloatHandle, "Dummy collection (output)"); declareProperty("vectorfloat", m_vectorFloatHandle, "Dummy collection (output)"); - declareProperty("singleint", m_singleIntHandle, "Dummy collection (output)"); } k4FWCoreTest_CreateExampleEventData::~k4FWCoreTest_CreateExampleEventData() {} @@ -29,11 +29,10 @@ StatusCode k4FWCoreTest_CreateExampleEventData::initialize() { } StatusCode k4FWCoreTest_CreateExampleEventData::execute() { - m_singleIntHandle.put(new int(12345)); - std::vector* floatVector = m_vectorFloatHandle.createAndPut(); - floatVector->emplace_back(125.); - floatVector->emplace_back(25.); + auto* floatVector = m_vectorFloatHandle.createAndPut(); + floatVector->push_back(125.); + floatVector->push_back(25.); edm4hep::MCParticleCollection* particles = m_mcParticleHandle.createAndPut(); diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h index 2e717100..8aa3929b 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h @@ -8,6 +8,9 @@ #include "TTree.h" #include "k4FWCore/DataHandle.h" +// podio +#include "podio/UserDataCollection.h" + // datamodel namespace edm4hep { class MCParticleCollection; @@ -54,8 +57,6 @@ class k4FWCoreTest_CreateExampleEventData : public GaudiAlgorithm { /// Handle for the Tracks to be written DataHandle m_trackHandle{"Tracks", Gaudi::DataHandle::Writer, this}; - DataHandle m_singleFloatHandle{"SingleFloat", Gaudi::DataHandle::Writer, this}; - DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; - DataHandle m_singleIntHandle{"SingleInt", Gaudi::DataHandle::Writer, this}; + DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; }; #endif /* K4FWCORE_K4FWCORETEST_CREATEEXAMPLEEVENTDATA */ diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp index c00bdf87..e19a2d60 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp @@ -22,12 +22,12 @@ StatusCode k4FWCoreTest_cellID_reader::execute() { const auto simtrackerhits_coll = m_simTrackerHitReaderHandle.get(); auto collID = simtrackerhits_coll->getID(); - const auto cellIDstr = m_simTrackerHitReaderHandle.getCollMetadataCellID(collID); - - if (cellIDstr != cellIDtest) { - std::cout << "ERROR cellID is: " << cellIDstr << std::endl; - return StatusCode::FAILURE; - } + // TODO - add a proper interface here + // const auto cellIDstr = m_simTrackerHitReaderHandle.getCollMetadataCellID(collID); + //if (cellIDstr != cellIDtest) { + // std::cout << "ERROR cellID is: " << cellIDstr << std::endl; + // return StatusCode::FAILURE; + //} return StatusCode::SUCCESS; } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp index 82fca559..afe80de2 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp @@ -31,8 +31,9 @@ StatusCode k4FWCoreTest_cellID_writer::execute() { auto hit = simTrackerHits->create(); hit.setPosition({3, 4, 5}); - auto& collmd = m_podioDataSvc->getProvider().getCollectionMetaData(simTrackerHits->getID()); - collmd.setValue("CellIDEncodingString", cellIDtest); + // TODO: decide on collection metadata interface in the data service + // something along the lines of + // m_podioDataSvc->addCollectionMetaData(collID, "name", value); return StatusCode::SUCCESS; } From cf46dac5ae00d5d74859fbe9ff90308dc54147ae Mon Sep 17 00:00:00 2001 From: hegner Date: Wed, 3 May 2023 11:17:22 +0200 Subject: [PATCH 02/24] Update k4FWCore/src/PodioDataSvc.cpp Co-authored-by: Thomas Madlener --- k4FWCore/src/PodioDataSvc.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index b20d29e5..3595a304 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -38,10 +38,6 @@ StatusCode PodioDataSvc::initialize() { } - // read - - // if empty, create a new frame from scratch - return status; } /// Service reinitialisation From 970d8bf1be633b87228aee9514246e877f2df944 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 3 May 2023 22:48:20 +0200 Subject: [PATCH 03/24] allow different event content for different events --- k4FWCore/components/PodioOutput.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/k4FWCore/components/PodioOutput.cpp b/k4FWCore/components/PodioOutput.cpp index 0bf718bc..a46c393e 100644 --- a/k4FWCore/components/PodioOutput.cpp +++ b/k4FWCore/components/PodioOutput.cpp @@ -30,11 +30,10 @@ StatusCode PodioOutput::initialize() { StatusCode PodioOutput::execute() { auto& frame = m_podioDataSvc->getFrame(); - // for now assume identical content for every event // register for writing + std::vector collection_names_to_write; if (m_firstEvent) { auto collections = frame.getAvailableCollections(); - std::vector collection_names_to_write; for (auto collection_name : collections) { if (m_switch.isOn(collection_name)) { collection_names_to_write.push_back(collection_name); @@ -42,7 +41,7 @@ StatusCode PodioOutput::execute() { } m_framewriter->writeFrame(frame, "events", collection_names_to_write); } else { - m_framewriter->writeFrame(frame, "events"); + m_framewriter->writeFrame(frame, "events", collection_names_to_write); } m_firstEvent = false; From f3f692811b6fe455495f9016870ab55aa7084287 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 4 May 2023 00:05:58 +0200 Subject: [PATCH 04/24] fix double bookkeeping of read collections --- k4FWCore/components/PodioInput.cpp | 7 ++++--- k4FWCore/components/PodioInput.h | 2 -- k4FWCore/src/PodioDataSvc.cpp | 9 +++------ 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/k4FWCore/components/PodioInput.cpp b/k4FWCore/components/PodioInput.cpp index 5df428ac..ced86fdd 100644 --- a/k4FWCore/components/PodioInput.cpp +++ b/k4FWCore/components/PodioInput.cpp @@ -27,13 +27,14 @@ StatusCode PodioInput::initialize() { StatusCode PodioInput::execute() { size_t cntr = 0; // Re-create the collections from ROOT file - for (auto& id : m_collectionIDs) { - const std::string& collName = m_collectionNames.value().at(cntr++); - debug() << "Registering collection to read " << collName << " with id " << id << endmsg; + + for (auto& collName : m_collectionNames) { + debug() << "Registering collection to read " << collName << endmsg; if (m_podioDataSvc->readCollection(collName).isFailure()) { return StatusCode::FAILURE; } } + // Tell data service that we are done with requested collections m_podioDataSvc->endOfRead(); return StatusCode::SUCCESS; diff --git a/k4FWCore/components/PodioInput.h b/k4FWCore/components/PodioInput.h index 4fb9b846..3bb31433 100644 --- a/k4FWCore/components/PodioInput.h +++ b/k4FWCore/components/PodioInput.h @@ -32,8 +32,6 @@ class PodioInput : public GaudiAlgorithm { private: /// Name of collections to read. Set by option collections (this is temporary) Gaudi::Property> m_collectionNames{this, "collections", {}, "Places of collections to read"}; - /// Collection IDs (retrieved with CollectionIDTable from ROOT file, using collection names) - std::vector m_collectionIDs; /// Data service: needed to register objects and get collection IDs. Just an observing pointer. PodioDataSvc* m_podioDataSvc; }; diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index 3595a304..36b0c58e 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -94,14 +94,11 @@ PodioDataSvc::~PodioDataSvc() {} StatusCode PodioDataSvc::readCollection(const std::string& collName) { const podio::CollectionBase* collection(nullptr); - collection = m_frame.get(collName); - if (collection->isSubsetCollection()) { - return StatusCode::SUCCESS; - } auto wrapper = new DataWrapper; - wrapper->setData(collection); - return registerObject("/Event", "/" + collName, wrapper); + wrapper->setData(collection); + m_podio_datawrappers.push_back(wrapper); + return DataSvc::registerObject("/Event", "/" + collName, wrapper); } StatusCode PodioDataSvc::registerObject(std::string_view parentPath, std::string_view fullPath, DataObject* pObject) { From dbd41d072a414ae7b845f65ad5b8180f6ea7ae79 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 4 May 2023 11:23:27 +0200 Subject: [PATCH 05/24] rename metadata frame to configuration_metadata --- k4FWCore/components/PodioOutput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4FWCore/components/PodioOutput.cpp b/k4FWCore/components/PodioOutput.cpp index a46c393e..34683758 100644 --- a/k4FWCore/components/PodioOutput.cpp +++ b/k4FWCore/components/PodioOutput.cpp @@ -93,7 +93,7 @@ StatusCode PodioOutput::finalize() { std::string s_env_key4hep_stack = env_key4hep_stack; metadata_frame.putParameter("key4hepstack", s_env_key4hep_stack); } - m_framewriter->writeFrame(metadata_frame, "metadata"); + m_framewriter->writeFrame(metadata_frame, "configuration_metadata"); // TODO: Create frames for run metadata and collection metadata From 70a46d2e00c05bdfff92ea98f36547f94f0f43dd Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 4 May 2023 12:47:34 +0200 Subject: [PATCH 06/24] make magic method in DataWrapper private; add proper friend declaration --- k4FWCore/include/k4FWCore/DataWrapper.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/k4FWCore/include/k4FWCore/DataWrapper.h b/k4FWCore/include/k4FWCore/DataWrapper.h index 289da335..f68635e0 100644 --- a/k4FWCore/include/k4FWCore/DataWrapper.h +++ b/k4FWCore/include/k4FWCore/DataWrapper.h @@ -7,6 +7,10 @@ #include "GaudiKernel/DataObject.h" #include "podio/CollectionBase.h" +// forward declaration +template +class DataHandle; + class GAUDI_API DataWrapperBase : public DataObject { public: // ugly hack to circumvent the usage of boost::any yet @@ -17,6 +21,9 @@ class GAUDI_API DataWrapperBase : public DataObject { }; template class GAUDI_API DataWrapper : public DataWrapperBase { +public: + template + friend class DataHandle; public: DataWrapper() : DataWrapperBase(), m_data(nullptr){}; virtual ~DataWrapper(); @@ -24,11 +31,14 @@ template class GAUDI_API DataWrapper : public DataWrapperBase { const T* getData() { return m_data; } void setData(const T* data) { m_data = data; } virtual void resetData() { m_data = nullptr; } + +private: /// try to cast to collectionBase; may return nullptr; virtual podio::CollectionBase* collectionBase(); private: const T* m_data; + }; template DataWrapper::~DataWrapper() { From c49bd178add2dc1a74c30b1f60b9edb0d91e4f11 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 4 May 2023 14:58:02 +0200 Subject: [PATCH 07/24] address more PR comments --- k4FWCore/components/PodioOutput.cpp | 9 ++++----- k4FWCore/components/PodioOutput.h | 3 ++- k4FWCore/include/k4FWCore/DataWrapper.h | 2 +- k4FWCore/src/PodioDataSvc.cpp | 3 +++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/k4FWCore/components/PodioOutput.cpp b/k4FWCore/components/PodioOutput.cpp index 34683758..3b8e0ab2 100644 --- a/k4FWCore/components/PodioOutput.cpp +++ b/k4FWCore/components/PodioOutput.cpp @@ -31,17 +31,16 @@ StatusCode PodioOutput::execute() { auto& frame = m_podioDataSvc->getFrame(); // register for writing - std::vector collection_names_to_write; if (m_firstEvent) { auto collections = frame.getAvailableCollections(); - for (auto collection_name : collections) { + for (auto& collection_name : collections) { if (m_switch.isOn(collection_name)) { - collection_names_to_write.push_back(collection_name); + m_collection_names_to_write.push_back(collection_name); } } - m_framewriter->writeFrame(frame, "events", collection_names_to_write); + m_framewriter->writeFrame(frame, "events", m_collection_names_to_write); } else { - m_framewriter->writeFrame(frame, "events", collection_names_to_write); + m_framewriter->writeFrame(frame, "events", m_collection_names_to_write); } m_firstEvent = false; diff --git a/k4FWCore/components/PodioOutput.h b/k4FWCore/components/PodioOutput.h index 7ca65715..6327ff74 100644 --- a/k4FWCore/components/PodioOutput.h +++ b/k4FWCore/components/PodioOutput.h @@ -37,12 +37,13 @@ class PodioOutput : public GaudiAlgorithm { "An optional file path to copy the outputfile to."}; /// Switch for keeping or dropping outputs KeepDropSwitch m_switch; - /// Needed for collection ID table PodioDataSvc* m_podioDataSvc; /// The actual ROOT frame writer std::unique_ptr m_framewriter; /// The stored collections std::vector m_storedCollections; + /// The collections to write out + std::vector m_collection_names_to_write; }; #endif diff --git a/k4FWCore/include/k4FWCore/DataWrapper.h b/k4FWCore/include/k4FWCore/DataWrapper.h index f68635e0..34755c17 100644 --- a/k4FWCore/include/k4FWCore/DataWrapper.h +++ b/k4FWCore/include/k4FWCore/DataWrapper.h @@ -48,7 +48,7 @@ template DataWrapper::~DataWrapper() { template podio::CollectionBase* DataWrapper::collectionBase() { if (std::is_base_of::value) { - return reinterpret_cast(const_cast(m_data)); + return const_cast(m_data); } return nullptr; } diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index 36b0c58e..b5e06aa7 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -95,6 +95,9 @@ PodioDataSvc::~PodioDataSvc() {} StatusCode PodioDataSvc::readCollection(const std::string& collName) { const podio::CollectionBase* collection(nullptr); collection = m_frame.get(collName); + if (collection == nullptr){ + error() << "Collection " << collName << " does not exist." << endmsg; + } auto wrapper = new DataWrapper; wrapper->setData(collection); m_podio_datawrappers.push_back(wrapper); From d0b60cb89da68077f53a9ead1314d9c99e6e686b Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 4 May 2023 15:27:00 +0200 Subject: [PATCH 08/24] add explicit podio requirement to CMake --- k4FWCore/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4FWCore/CMakeLists.txt b/k4FWCore/CMakeLists.txt index 0b3535a6..b1b27c9f 100644 --- a/k4FWCore/CMakeLists.txt +++ b/k4FWCore/CMakeLists.txt @@ -2,7 +2,7 @@ # Package: k4FWCore ################################################################################ -find_package(podio) +find_package(podio 0.16.3 REQUIRED) gaudi_install(SCRIPTS) From 5f797e8e56cd3c3336d6580ca21c31e7c2be50ff Mon Sep 17 00:00:00 2001 From: tmadlener Date: Sun, 7 May 2023 13:01:35 -0400 Subject: [PATCH 09/24] Add a failing test case that shows the issue with Frame I/O --- test/k4FWCoreTest/CMakeLists.txt | 21 ++++++-- .../options/checkExampleEventData.py | 33 +++++++++++++ .../k4FWCoreTest_CheckExampleEventData.cpp | 43 +++++++++++++++++ .../k4FWCoreTest_CheckExampleEventData.h | 48 +++++++++++++++++++ .../k4FWCoreTest_CreateExampleEventData.cpp | 8 ++-- .../k4FWCoreTest_CreateExampleEventData.h | 2 + 6 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 test/k4FWCoreTest/options/checkExampleEventData.py create mode 100644 test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp create mode 100644 test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index d2fd749b..d736b1c8 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -4,10 +4,17 @@ find_package(EDM4HEP) +set(k4fwcoretest_plugin_sources + src/components/TestDataHandleUniquePtr.cpp + src/components/TestUniqueIDGenSvc.cpp + src/components/k4FWCoreTest_AlgorithmWithTFile.cpp + src/components/k4FWCoreTest_CreateExampleEventData.cpp + src/components/k4FWCoreTest_HelloWorldAlg.cpp + src/components/k4FWCoreTest_cellID_reader.cpp + src/components/k4FWCoreTest_cellID_writer.cpp + src/components/k4FWCoreTest_CheckExampleEventData.cpp +) - - -file(GLOB k4fwcoretest_plugin_sources src/components/*.cpp) gaudi_add_module(k4FWCoreTestPlugins SOURCES ${k4fwcoretest_plugin_sources} LINK Gaudi::GaudiAlgLib Gaudi::GaudiKernel k4FWCore k4FWCore::k4Interface ROOT::Core ROOT::RIO ROOT::Tree EDM4HEP::edm4hepDict EDM4HEP::edm4hep) @@ -45,6 +52,12 @@ add_test(NAME CreateExampleEventData COMMAND ${K4RUN} options/createExampleEventData.py) set_test_env(CreateExampleEventData) +add_test(NAME CheckExampleEventData + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND ${K4RUN} options/checkExampleEventData.py +) +set_test_env(CheckExampleEventData) + add_test(NAME CreateExampleEventData_cellID WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND ${K4RUN} options/createExampleEventData_cellID.py) @@ -111,4 +124,4 @@ set_test_env(TestUniqueIDGenSvc) add_test(NAME CreateLegacyExampleEventData WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND ${K4RUN} options/createLegacyExampleEventData.py) -set_test_env(CreateLegacyExampleEventData) \ No newline at end of file +set_test_env(CreateLegacyExampleEventData) diff --git a/test/k4FWCoreTest/options/checkExampleEventData.py b/test/k4FWCoreTest/options/checkExampleEventData.py new file mode 100644 index 00000000..9b75d977 --- /dev/null +++ b/test/k4FWCoreTest/options/checkExampleEventData.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +from Gaudi.Configuration import * + +from Configurables import k4DataSvc + +podioevent = k4DataSvc("EventDataSvc") +podioevent.input = "output_k4test_exampledata.root" + +from Configurables import PodioInput + +inp = PodioInput() +inp.collections = [ + "VectorFloat", + "MCParticles", + "SimTrackerHits", + "TrackerHits", + "Tracks", +] + +from Configurables import k4FWCoreTest_CheckExampleEventData + +checker = k4FWCoreTest_CheckExampleEventData() + +from Configurables import ApplicationMgr + +ApplicationMgr( + TopAlg=[inp, checker], + EvtSel="NONE", + EvtMax=-1, + ExtSvc=[podioevent], + OutputLevel=INFO, + StopOnSignal=True, +) diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp new file mode 100644 index 00000000..cc72cae3 --- /dev/null +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp @@ -0,0 +1,43 @@ +#include "k4FWCoreTest_CheckExampleEventData.h" +#include + +// datamodel +#include "edm4hep/MCParticleCollection.h" +#include "edm4hep/SimTrackerHitCollection.h" +#include "edm4hep/TrackCollection.h" +#include "edm4hep/TrackerHitCollection.h" + +DECLARE_COMPONENT(k4FWCoreTest_CheckExampleEventData) + +k4FWCoreTest_CheckExampleEventData::k4FWCoreTest_CheckExampleEventData(const std::string& aName, ISvcLocator* aSvcLoc) + : GaudiAlgorithm(aName, aSvcLoc) { + declareProperty("mcparticles", m_mcParticleHandle, "Dummy Particle collection (output)"); + declareProperty("vectorfloat", m_vectorFloatHandle, "Dummy collection (output)"); +} + +StatusCode k4FWCoreTest_CheckExampleEventData::initialize() { return GaudiAlgorithm::initialize(); } + +StatusCode k4FWCoreTest_CheckExampleEventData::execute() { + auto floatVector = m_vectorFloatHandle.get(); + if (floatVector->size() != 3 || (*floatVector)[2] != m_event) { + fatal() << "Contents of vectorfloat collection is not as expected: size = " << floatVector->size() + << " (expected 3), contents = " << *floatVector << " (expected [125., 25., " << m_event << "]) " + << std::endl; + return StatusCode::FAILURE; + } + + auto particles = m_mcParticleHandle.get(); + auto particle = (*particles)[0]; + if ((particle.getMomentum().x != m_magicNumberOffset + m_event + 5) || + (particle.getMass() != m_magicNumberOffset + m_event + 8)) { + fatal() << "Contents of mcparticles collection is not as expected: momentum.x = " << particle.getMomentum().x + << " (expected " << m_magicNumberOffset + m_event + 5 << "), mass = " << particle.getMass() << " (expected " + << m_magicNumberOffset + m_event + 8 << ")" << std::endl; + return StatusCode::FAILURE; + } + + m_event++; + return StatusCode::SUCCESS; +} + +StatusCode k4FWCoreTest_CheckExampleEventData::finalize() { return GaudiAlgorithm::finalize(); } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h new file mode 100644 index 00000000..e4e5447c --- /dev/null +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h @@ -0,0 +1,48 @@ +#ifndef K4FWCORE_K4FWCORETEST_CHECKEEXAMPLEEVENTDATA +#define K4FWCORE_K4FWCORETEST_CHECKEEXAMPLEEVENTDATA + +#include "k4FWCore/DataHandle.h" + +// GAUDI +#include "GaudiAlg/GaudiAlgorithm.h" + +#include "podio/UserDataCollection.h" + +// datamodel +namespace edm4hep { + class MCParticleCollection; + class SimTrackerHitCollection; + class TrackerHitCollection; + class SimCaloHitCollection; + class TrackCollection; +} // namespace edm4hep + +class k4FWCoreTest_CheckExampleEventData : public GaudiAlgorithm { +public: + explicit k4FWCoreTest_CheckExampleEventData(const std::string&, ISvcLocator*); + ~k4FWCoreTest_CheckExampleEventData() = default; + /** Initialize. + * @return status code + */ + virtual StatusCode initialize() final; + /** Execute. + * @return status code + */ + virtual StatusCode execute() final; + /** Finalize. + * @return status code + */ + virtual StatusCode finalize() final; + +private: + /// integer to add to the dummy values written to the edm + Gaudi::Property m_magicNumberOffset{this, "magicNumberOffset", 0, + "Integer to add to the dummy values written to the edm"}; + /// Handle for the MCParticles to be written + DataHandle m_mcParticleHandle{"MCParticles", Gaudi::DataHandle::Reader, this}; + DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Reader, this}; + + int m_event{0}; +}; + +#endif // K4FWCORE_K4FWCORETEST_CHECKEEXAMPLEEVENTDATA diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp index da742de7..4ace0bea 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp @@ -29,20 +29,20 @@ StatusCode k4FWCoreTest_CreateExampleEventData::initialize() { } StatusCode k4FWCoreTest_CreateExampleEventData::execute() { - auto* floatVector = m_vectorFloatHandle.createAndPut(); floatVector->push_back(125.); floatVector->push_back(25.); + floatVector->push_back(m_event); edm4hep::MCParticleCollection* particles = m_mcParticleHandle.createAndPut(); auto particle = particles->create(); auto& p4 = particle.momentum(); - p4.x = m_magicNumberOffset + 5; + p4.x = m_magicNumberOffset + m_event + 5; p4.y = m_magicNumberOffset + 6; p4.z = m_magicNumberOffset + 7; - particle.setMass(m_magicNumberOffset + 8); + particle.setMass(m_magicNumberOffset + m_event + 8); edm4hep::SimTrackerHitCollection* simTrackerHits = m_simTrackerHitHandle.createAndPut(); auto hit = simTrackerHits->create(); @@ -70,6 +70,8 @@ StatusCode k4FWCoreTest_CreateExampleEventData::execute() { track.addToTrackerHits(trackerHit); track.addToTracks(track2); + m_event++; + return StatusCode::SUCCESS; } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h index 8aa3929b..75b33d19 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h @@ -58,5 +58,7 @@ class k4FWCoreTest_CreateExampleEventData : public GaudiAlgorithm { DataHandle m_trackHandle{"Tracks", Gaudi::DataHandle::Writer, this}; DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; + + int m_event{0}; }; #endif /* K4FWCORE_K4FWCORETEST_CREATEEXAMPLEEVENTDATA */ From 4fcfdb16913d121f5c7fbaf26b16d445c90445e6 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 10 May 2023 18:44:06 +0200 Subject: [PATCH 10/24] move reading/creation of frames to i_setRoot --- k4FWCore/include/k4FWCore/PodioDataSvc.h | 2 ++ k4FWCore/src/PodioDataSvc.cpp | 27 ++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index db27c03c..4b93a18f 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -26,6 +26,8 @@ class PodioDataSvc : public DataSvc { virtual StatusCode reinitialize(); virtual StatusCode finalize(); virtual StatusCode clearStore(); + virtual StatusCode i_setRoot( std::string root_path, IOpaqueAddress* pRootAddr ); + virtual StatusCode i_setRoot( std::string root_path, DataObject* pRootObj ); /// Standard Constructor PodioDataSvc(const std::string& name, ISvcLocator* svc); diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index b5e06aa7..661dbad5 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -30,12 +30,7 @@ StatusCode PodioDataSvc::initialize() { if (m_1stEvtEntry != 0) { m_eventMax -= m_1stEvtEntry; } - - m_frame = podio::Frame(m_reader.readEntry("events",m_1stEvtEntry)); } - } else { - m_frame = podio::Frame(); - } return status; @@ -60,21 +55,35 @@ StatusCode PodioDataSvc::clearStore() { } m_podio_datawrappers.clear(); + DataSvc::clearStore().ignore(); + return StatusCode::SUCCESS; +} + +StatusCode PodioDataSvc::i_setRoot(std::string root_path, IOpaqueAddress* pRootAddr) { // create a new frame if (m_reading_from_file) { - m_frame = podio::Frame(m_reader.readNextEntry("events")); + m_frame = podio::Frame(m_reader.readEntry("events", m_eventNum + m_1stEvtEntry)); } else { m_frame = podio::Frame(); } + return DataSvc::i_setRoot(root_path, pRootAddr); +} - DataSvc::clearStore().ignore(); - return StatusCode::SUCCESS; +StatusCode PodioDataSvc::i_setRoot(std::string root_path, DataObject* pRootObj) { + // create a new frame + if (m_reading_from_file) { + m_frame = podio::Frame(m_reader.readEntry("events", m_eventNum + m_1stEvtEntry)); + } else { + m_frame = podio::Frame(); + } + return DataSvc::i_setRoot(root_path, pRootObj); } + void PodioDataSvc::endOfRead() { StatusCode sc; if (m_eventMax != -1) { - if (m_eventNum++ > m_eventMax) { + if (m_eventNum++ >= m_eventMax-1) { // we start counting at 0 thus the -1. info() << "Reached end of file with event " << m_eventMax << endmsg; IEventProcessor* eventProcessor; sc = service("ApplicationMgr", eventProcessor); From 4964a3b1875bc8a187ddae5b8d03be628c18b7b8 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Fri, 12 May 2023 14:03:05 +0200 Subject: [PATCH 11/24] adjust ctest for return code of Gaudi in ScheduledStop --- test/k4FWCoreTest/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index d736b1c8..c87ed855 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -57,6 +57,8 @@ add_test(NAME CheckExampleEventData COMMAND ${K4RUN} options/checkExampleEventData.py ) set_test_env(CheckExampleEventData) +set_tests_properties( CheckExampleEventData + PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop") add_test(NAME CreateExampleEventData_cellID WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} From db4aac78c9b35d9fad9fe70ea37ab5dfa10b631e Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Mon, 22 May 2023 11:18:11 +0200 Subject: [PATCH 12/24] enable transient non-PODIO types --- k4FWCore/include/k4FWCore/DataHandle.h | 5 +---- k4FWCore/include/k4FWCore/DataWrapper.h | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/k4FWCore/include/k4FWCore/DataHandle.h b/k4FWCore/include/k4FWCore/DataHandle.h index 3499d06d..bb604780 100644 --- a/k4FWCore/include/k4FWCore/DataHandle.h +++ b/k4FWCore/include/k4FWCore/DataHandle.h @@ -74,10 +74,7 @@ DataHandle::DataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode podio_data_service = dynamic_cast(m_eds.get()); if (nullptr != podio_data_service) { - if (!std::is_convertible::value) { - throw GaudiException("DataHandle", "Trying to add non-PODIO type to data service", StatusCode::FAILURE); - - } else { m_dataPtr = new T();} + m_dataPtr = new T(); } else { // This is the legacy implementation kept for a transition period diff --git a/k4FWCore/include/k4FWCore/DataWrapper.h b/k4FWCore/include/k4FWCore/DataWrapper.h index 34755c17..04476532 100644 --- a/k4FWCore/include/k4FWCore/DataWrapper.h +++ b/k4FWCore/include/k4FWCore/DataWrapper.h @@ -47,7 +47,7 @@ template DataWrapper::~DataWrapper() { } template podio::CollectionBase* DataWrapper::collectionBase() { - if (std::is_base_of::value) { + if constexpr (std::is_base_of::value) { return const_cast(m_data); } return nullptr; From 12a916a5fc4e6733bf9bab54a7630141bad8eb38 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Mon, 22 May 2023 11:55:26 +0200 Subject: [PATCH 13/24] Fix final missing constexpr to make transient store work --- k4FWCore/include/k4FWCore/DataHandle.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4FWCore/include/k4FWCore/DataHandle.h b/k4FWCore/include/k4FWCore/DataHandle.h index bb604780..cdbd135d 100644 --- a/k4FWCore/include/k4FWCore/DataHandle.h +++ b/k4FWCore/include/k4FWCore/DataHandle.h @@ -81,7 +81,7 @@ DataHandle::DataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode PodioLegacyDataSvc* plds; plds = dynamic_cast(m_eds.get()); if (nullptr != plds) { - if (std::is_convertible::value) { + if constexpr (std::is_convertible::value) { // case 1: T is a podio collection // for this case creation of branches is still handled in PodioOutput // (but could be moved here in the future) From 4a8f7ae17eae75bff92a6792b4bbacc82baa4451 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 24 May 2023 13:40:07 +0200 Subject: [PATCH 14/24] add prototype for MetaDataHandle; no R/W mode implementation yet --- k4FWCore/components/PodioOutput.cpp | 13 +-- k4FWCore/include/k4FWCore/MetaDataHandle.h | 94 +++++++++++++++++++ k4FWCore/include/k4FWCore/PodioDataSvc.h | 7 +- k4FWCore/src/PodioDataSvc.cpp | 18 ++-- .../k4FWCoreTest_CheckExampleEventData.cpp | 1 + .../k4FWCoreTest_CheckExampleEventData.h | 5 +- .../k4FWCoreTest_CreateExampleEventData.cpp | 1 + .../k4FWCoreTest_CreateExampleEventData.h | 6 +- 8 files changed, 129 insertions(+), 16 deletions(-) create mode 100644 k4FWCore/include/k4FWCore/MetaDataHandle.h diff --git a/k4FWCore/components/PodioOutput.cpp b/k4FWCore/components/PodioOutput.cpp index 3b8e0ab2..2b3ac519 100644 --- a/k4FWCore/components/PodioOutput.cpp +++ b/k4FWCore/components/PodioOutput.cpp @@ -28,7 +28,7 @@ StatusCode PodioOutput::initialize() { } StatusCode PodioOutput::execute() { - auto& frame = m_podioDataSvc->getFrame(); + auto& frame = m_podioDataSvc->getEventFrame(); // register for writing if (m_firstEvent) { @@ -86,15 +86,16 @@ StatusCode PodioOutput::finalize() { } // Collect all the metadata - podio::Frame metadata_frame{}; - metadata_frame.putParameter("gaudiConfigOptions", config_data); + podio::Frame config_metadata_frame{}; + config_metadata_frame.putParameter("gaudiConfigOptions", config_data); if (const char* env_key4hep_stack = std::getenv("KEY4HEP_STACK")) { std::string s_env_key4hep_stack = env_key4hep_stack; - metadata_frame.putParameter("key4hepstack", s_env_key4hep_stack); + config_metadata_frame.putParameter("key4hepstack", s_env_key4hep_stack); } - m_framewriter->writeFrame(metadata_frame, "configuration_metadata"); + m_framewriter->writeFrame(config_metadata_frame, "configuration_metadata"); - // TODO: Create frames for run metadata and collection metadata + auto& metadata_frame = m_podioDataSvc->getMetaDataFrame(); + m_framewriter->writeFrame(metadata_frame, "metadata"); // write information into file m_framewriter->finish(); diff --git a/k4FWCore/include/k4FWCore/MetaDataHandle.h b/k4FWCore/include/k4FWCore/MetaDataHandle.h new file mode 100644 index 00000000..9505f10f --- /dev/null +++ b/k4FWCore/include/k4FWCore/MetaDataHandle.h @@ -0,0 +1,94 @@ +#ifndef K4FWCORE_METADATAHANDLE_H +#define K4FWCORE_METADATAHANDLE_H + +// GAUDI +#include "GaudiAlg/GaudiAlgorithm.h" + +#include "podio/GenericParameters.h" +#include "k4FWCore/PodioDataSvc.h" + +#include "GaudiKernel/MsgStream.h" + +#include + +template class MetaDataHandle { +public: + MetaDataHandle(); + MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); + MetaDataHandle(const Gaudi::DataHandle* handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); + ~MetaDataHandle(); + + const T get(); + void put(T); + +private: + std::string fullDescriptor(); + +private: + ServiceHandle m_eds; + std::string m_descriptor; + PodioDataSvc* m_podio_data_service{nullptr}; + Algorithm* m_fatherAlg{nullptr}; + const Gaudi::DataHandle* m_dataHandle{nullptr}; // holds the identifier in case we do collection metadata +}; + +template MetaDataHandle::~MetaDataHandle() { +} + +//--------------------------------------------------------------------------- +template +MetaDataHandle::MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) +: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg) { + + StatusCode sc = m_eds.retrieve(); + m_podio_data_service = dynamic_cast(m_eds.get()); + if (nullptr == m_podio_data_service) { + m_fatherAlg->fatal() << "MetaDataHandles require the PodioDataSvc" << endmsg; + } +} + +//--------------------------------------------------------------------------- +template +MetaDataHandle::MetaDataHandle(const Gaudi::DataHandle* handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) +: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg), m_dataHandle{handle} { + + StatusCode sc = m_eds.retrieve(); + m_podio_data_service = dynamic_cast(m_eds.get()); + if (nullptr == m_podio_data_service) { + m_fatherAlg->fatal() << "MetaDataHandles require the PodioDataSvc" << endmsg; + } +} + + +//--------------------------------------------------------------------------- +template const T MetaDataHandle::get() { + podio::Frame& frame = m_podio_data_service->getMetaDataFrame(); + return frame.getParameter(fullDescriptor()); +} + +//--------------------------------------------------------------------------- +template void MetaDataHandle::put(T value) { + + // check whether we are in the proper State + // put is only allowed in the initalization + if (m_podio_data_service->targetFSMState() != Gaudi::StateMachine::INITIALIZED) { + m_fatherAlg->fatal() << "A MetaDataHandle::put can only be used during initialize" << endmsg; + } + std::string full_descriptor = fullDescriptor(); + podio::Frame& frame = m_podio_data_service->getMetaDataFrame(); + frame.putParameter(full_descriptor, value); +} + +//--------------------------------------------------------------------------- +template std::string MetaDataHandle::fullDescriptor() { + + std::string full_descriptor; + if (nullptr != m_dataHandle) { + full_descriptor = m_dataHandle->objKey() + "__" + m_descriptor; + full_descriptor.erase(0,7); // remove the "/Event/" part of the collections' object key + } else { + full_descriptor = m_descriptor; + } + return full_descriptor; +} +#endif diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index 4b93a18f..05d0bdf6 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -44,7 +44,8 @@ class PodioDataSvc : public DataSvc { StatusCode readCollection(const std::string& collectionName); - const podio::Frame& getFrame() const { return m_frame; } + const podio::Frame& getEventFrame() const { return m_eventframe; } + podio::Frame& getMetaDataFrame() { return m_metadataframe; } /// Resets caches of reader and event store, increases event counter void endOfRead(); @@ -53,7 +54,9 @@ class PodioDataSvc : public DataSvc { /// PODIO reader for ROOT files podio::ROOTFrameReader m_reader; /// PODIO Frame, used to initialise collections - podio::Frame m_frame; + podio::Frame m_eventframe; + /// PODIO Frame, used to store metadata + podio::Frame m_metadataframe; /// Counter of the event number int m_eventNum{0}; /// Number of events in the file / to process diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index 661dbad5..822ee687 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -33,6 +33,12 @@ StatusCode PodioDataSvc::initialize() { } } + if (m_reading_from_file == true) { + m_metadataframe = m_reader.readEntry("metadata",0); + } else { + m_metadataframe = podio::Frame(); + } + return status; } /// Service reinitialisation @@ -62,9 +68,9 @@ StatusCode PodioDataSvc::clearStore() { StatusCode PodioDataSvc::i_setRoot(std::string root_path, IOpaqueAddress* pRootAddr) { // create a new frame if (m_reading_from_file) { - m_frame = podio::Frame(m_reader.readEntry("events", m_eventNum + m_1stEvtEntry)); + m_eventframe = podio::Frame(m_reader.readEntry("events", m_eventNum + m_1stEvtEntry)); } else { - m_frame = podio::Frame(); + m_eventframe = podio::Frame(); } return DataSvc::i_setRoot(root_path, pRootAddr); } @@ -72,9 +78,9 @@ StatusCode PodioDataSvc::i_setRoot(std::string root_path, IOpaqueAddress* pRootA StatusCode PodioDataSvc::i_setRoot(std::string root_path, DataObject* pRootObj) { // create a new frame if (m_reading_from_file) { - m_frame = podio::Frame(m_reader.readEntry("events", m_eventNum + m_1stEvtEntry)); + m_eventframe = podio::Frame(m_reader.readEntry("events", m_eventNum + m_1stEvtEntry)); } else { - m_frame = podio::Frame(); + m_eventframe = podio::Frame(); } return DataSvc::i_setRoot(root_path, pRootObj); } @@ -103,7 +109,7 @@ PodioDataSvc::~PodioDataSvc() {} StatusCode PodioDataSvc::readCollection(const std::string& collName) { const podio::CollectionBase* collection(nullptr); - collection = m_frame.get(collName); + collection = m_eventframe.get(collName); if (collection == nullptr){ error() << "Collection " << collName << " does not exist." << endmsg; } @@ -121,7 +127,7 @@ StatusCode PodioDataSvc::registerObject(std::string_view parentPath, std::string size_t pos = fullPath.find_last_of("/"); std::string shortPath(fullPath.substr(pos + 1, fullPath.length())); // Attention: this passes the ownership of the data to the frame - m_frame.put(std::unique_ptr(coll), shortPath); + m_eventframe.put(std::unique_ptr(coll), shortPath); m_podio_datawrappers.push_back(wrapper); } } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp index cc72cae3..f96dbcf6 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp @@ -18,6 +18,7 @@ k4FWCoreTest_CheckExampleEventData::k4FWCoreTest_CheckExampleEventData(const std StatusCode k4FWCoreTest_CheckExampleEventData::initialize() { return GaudiAlgorithm::initialize(); } StatusCode k4FWCoreTest_CheckExampleEventData::execute() { + auto metadata = m_metadatafloat.get(); auto floatVector = m_vectorFloatHandle.get(); if (floatVector->size() != 3 || (*floatVector)[2] != m_event) { fatal() << "Contents of vectorfloat collection is not as expected: size = " << floatVector->size() diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h index e4e5447c..48037018 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h @@ -2,7 +2,7 @@ #define K4FWCORE_K4FWCORETEST_CHECKEEXAMPLEEVENTDATA #include "k4FWCore/DataHandle.h" - +#include "k4FWCore/MetaDataHandle.h" // GAUDI #include "GaudiAlg/GaudiAlgorithm.h" @@ -42,6 +42,9 @@ class k4FWCoreTest_CheckExampleEventData : public GaudiAlgorithm { DataHandle m_mcParticleHandle{"MCParticles", Gaudi::DataHandle::Reader, this}; DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Reader, this}; + // Some test MetaData + MetaDataHandle m_metadatafloat{&m_vectorFloatHandle, "someCollectionMetaData", Gaudi::DataHandle::Writer, this}; + int m_event{0}; }; diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp index 4ace0bea..c2418d17 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp @@ -25,6 +25,7 @@ StatusCode k4FWCoreTest_CreateExampleEventData::initialize() { if (GaudiAlgorithm::initialize().isFailure()) { return StatusCode::FAILURE; } + m_metadatafloat.put(4.); return StatusCode::SUCCESS; } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h index 75b33d19..0b9383cb 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h @@ -4,9 +4,10 @@ // GAUDI #include "GaudiAlg/GaudiAlgorithm.h" -// edm4hep +// key4hep #include "TTree.h" #include "k4FWCore/DataHandle.h" +#include "k4FWCore/MetaDataHandle.h" // podio #include "podio/UserDataCollection.h" @@ -59,6 +60,9 @@ class k4FWCoreTest_CreateExampleEventData : public GaudiAlgorithm { DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; + // Some test MetaData + MetaDataHandle m_metadatafloat{&m_vectorFloatHandle, "someCollectionMetaData", Gaudi::DataHandle::Writer, this}; + int m_event{0}; }; #endif /* K4FWCORE_K4FWCORETEST_CREATEEXAMPLEEVENTDATA */ From fdbddab0a3f3c0f06a93e8aaec2f891871b904a2 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 24 May 2023 22:30:07 +0200 Subject: [PATCH 15/24] include feedback to MetaDataHandle --- k4FWCore/include/k4FWCore/MetaDataHandle.h | 23 ++++++++++++------- k4FWCore/include/k4FWCore/PodioDataSvc.h | 9 +++++++- .../k4FWCoreTest_CheckExampleEventData.cpp | 1 - .../k4FWCoreTest_CheckExampleEventData.h | 5 +--- .../k4FWCoreTest_CreateExampleEventData.cpp | 1 - .../k4FWCoreTest_CreateExampleEventData.h | 5 ---- .../components/k4FWCoreTest_cellID_reader.cpp | 11 ++++----- .../components/k4FWCoreTest_cellID_reader.h | 7 +++--- .../components/k4FWCoreTest_cellID_writer.cpp | 17 +++----------- .../components/k4FWCoreTest_cellID_writer.h | 8 +++---- 10 files changed, 40 insertions(+), 47 deletions(-) diff --git a/k4FWCore/include/k4FWCore/MetaDataHandle.h b/k4FWCore/include/k4FWCore/MetaDataHandle.h index 9505f10f..2e4b83be 100644 --- a/k4FWCore/include/k4FWCore/MetaDataHandle.h +++ b/k4FWCore/include/k4FWCore/MetaDataHandle.h @@ -15,7 +15,7 @@ template class MetaDataHandle { public: MetaDataHandle(); MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); - MetaDataHandle(const Gaudi::DataHandle* handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); + MetaDataHandle(const Gaudi::DataHandle& handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); ~MetaDataHandle(); const T get(); @@ -29,7 +29,9 @@ template class MetaDataHandle { std::string m_descriptor; PodioDataSvc* m_podio_data_service{nullptr}; Algorithm* m_fatherAlg{nullptr}; - const Gaudi::DataHandle* m_dataHandle{nullptr}; // holds the identifier in case we do collection metadata + const Gaudi::DataHandle* m_dataHandle{nullptr}; // holds the identifier in case we do collection metadata + Gaudi::DataHandle::Mode m_mode; + }; template MetaDataHandle::~MetaDataHandle() { @@ -38,7 +40,7 @@ template MetaDataHandle::~MetaDataHandle() { //--------------------------------------------------------------------------- template MetaDataHandle::MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) -: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg) { +: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg), m_mode(a) { StatusCode sc = m_eds.retrieve(); m_podio_data_service = dynamic_cast(m_eds.get()); @@ -49,8 +51,8 @@ MetaDataHandle::MetaDataHandle(const std::string& descriptor, Gaudi::DataHand //--------------------------------------------------------------------------- template -MetaDataHandle::MetaDataHandle(const Gaudi::DataHandle* handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) -: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg), m_dataHandle{handle} { +MetaDataHandle::MetaDataHandle(const Gaudi::DataHandle& handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) +: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg), m_dataHandle(&handle), m_mode(a) { StatusCode sc = m_eds.retrieve(); m_podio_data_service = dynamic_cast(m_eds.get()); @@ -69,10 +71,12 @@ template const T MetaDataHandle::get() { //--------------------------------------------------------------------------- template void MetaDataHandle::put(T value) { + if (m_mode != Gaudi::DataHandle::Writer) + throw GaudiException("MetaDataHandle policy violation", "Put for non-writing MetaDataHandle not allowed", StatusCode::FAILURE); // check whether we are in the proper State // put is only allowed in the initalization - if (m_podio_data_service->targetFSMState() != Gaudi::StateMachine::INITIALIZED) { - m_fatherAlg->fatal() << "A MetaDataHandle::put can only be used during initialize" << endmsg; + if (m_podio_data_service->targetFSMState() == Gaudi::StateMachine::RUNNING) { + m_fatherAlg->fatal() << "A MetaDataHandle::put cannot be used during the event loop" << endmsg; } std::string full_descriptor = fullDescriptor(); podio::Frame& frame = m_podio_data_service->getMetaDataFrame(); @@ -85,7 +89,10 @@ template std::string MetaDataHandle::fullDescriptor() { std::string full_descriptor; if (nullptr != m_dataHandle) { full_descriptor = m_dataHandle->objKey() + "__" + m_descriptor; - full_descriptor.erase(0,7); // remove the "/Event/" part of the collections' object key + // remove the "/Event/" part of the collections' object key if in read mode + if (m_mode == Gaudi::DataHandle::Reader) { + full_descriptor.erase(0,7); + } } else { full_descriptor = m_descriptor; } diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index 05d0bdf6..04902461 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -11,6 +11,8 @@ #include // Forward declarations class DataWrapperBase; +class PodioOutput; +template class MetaDataHandle; /** @class PodioEvtSvc EvtDataSvc.h * @@ -19,6 +21,9 @@ class DataWrapperBase; * @author B. Hegner */ class PodioDataSvc : public DataSvc { + template + friend class MetaDataHandle; + friend class PodioOutput; public: typedef std::vector> CollRegistry; @@ -45,11 +50,13 @@ class PodioDataSvc : public DataSvc { StatusCode readCollection(const std::string& collectionName); const podio::Frame& getEventFrame() const { return m_eventframe; } - podio::Frame& getMetaDataFrame() { return m_metadataframe; } /// Resets caches of reader and event store, increases event counter void endOfRead(); +private: + podio::Frame& getMetaDataFrame() { return m_metadataframe; } + private: /// PODIO reader for ROOT files podio::ROOTFrameReader m_reader; diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp index f96dbcf6..cc72cae3 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp @@ -18,7 +18,6 @@ k4FWCoreTest_CheckExampleEventData::k4FWCoreTest_CheckExampleEventData(const std StatusCode k4FWCoreTest_CheckExampleEventData::initialize() { return GaudiAlgorithm::initialize(); } StatusCode k4FWCoreTest_CheckExampleEventData::execute() { - auto metadata = m_metadatafloat.get(); auto floatVector = m_vectorFloatHandle.get(); if (floatVector->size() != 3 || (*floatVector)[2] != m_event) { fatal() << "Contents of vectorfloat collection is not as expected: size = " << floatVector->size() diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h index 48037018..e4e5447c 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.h @@ -2,7 +2,7 @@ #define K4FWCORE_K4FWCORETEST_CHECKEEXAMPLEEVENTDATA #include "k4FWCore/DataHandle.h" -#include "k4FWCore/MetaDataHandle.h" + // GAUDI #include "GaudiAlg/GaudiAlgorithm.h" @@ -42,9 +42,6 @@ class k4FWCoreTest_CheckExampleEventData : public GaudiAlgorithm { DataHandle m_mcParticleHandle{"MCParticles", Gaudi::DataHandle::Reader, this}; DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Reader, this}; - // Some test MetaData - MetaDataHandle m_metadatafloat{&m_vectorFloatHandle, "someCollectionMetaData", Gaudi::DataHandle::Writer, this}; - int m_event{0}; }; diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp index c2418d17..4ace0bea 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp @@ -25,7 +25,6 @@ StatusCode k4FWCoreTest_CreateExampleEventData::initialize() { if (GaudiAlgorithm::initialize().isFailure()) { return StatusCode::FAILURE; } - m_metadatafloat.put(4.); return StatusCode::SUCCESS; } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h index 0b9383cb..a85f4cd1 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.h @@ -5,9 +5,7 @@ #include "GaudiAlg/GaudiAlgorithm.h" // key4hep -#include "TTree.h" #include "k4FWCore/DataHandle.h" -#include "k4FWCore/MetaDataHandle.h" // podio #include "podio/UserDataCollection.h" @@ -60,9 +58,6 @@ class k4FWCoreTest_CreateExampleEventData : public GaudiAlgorithm { DataHandle> m_vectorFloatHandle{"VectorFloat", Gaudi::DataHandle::Writer, this}; - // Some test MetaData - MetaDataHandle m_metadatafloat{&m_vectorFloatHandle, "someCollectionMetaData", Gaudi::DataHandle::Writer, this}; - int m_event{0}; }; #endif /* K4FWCORE_K4FWCORETEST_CREATEEXAMPLEEVENTDATA */ diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp index e19a2d60..01c187e1 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.cpp @@ -22,12 +22,11 @@ StatusCode k4FWCoreTest_cellID_reader::execute() { const auto simtrackerhits_coll = m_simTrackerHitReaderHandle.get(); auto collID = simtrackerhits_coll->getID(); - // TODO - add a proper interface here - // const auto cellIDstr = m_simTrackerHitReaderHandle.getCollMetadataCellID(collID); - //if (cellIDstr != cellIDtest) { - // std::cout << "ERROR cellID is: " << cellIDstr << std::endl; - // return StatusCode::FAILURE; - //} + const auto cellIDstr = m_cellIDHandle.get(); + if (cellIDstr != cellIDtest) { + error() << "ERROR cellID is: " << cellIDstr << endmsg; + return StatusCode::FAILURE; + } return StatusCode::SUCCESS; } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h index 218298f9..48dc6d60 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h @@ -4,15 +4,15 @@ // GAUDI #include "GaudiAlg/GaudiAlgorithm.h" -// edm4hep -#include "TTree.h" +// key4hep #include "k4FWCore/DataHandle.h" +#include "k4FWCore/MetaDataHandle.h" // datamodel #include "edm4hep/SimTrackerHitCollection.h" /** @class k4FWCoreTest_cellID - * Lightweight producer for edm data to test cellID reading + * Lightweight reader for edm data to test cellID reading */ class k4FWCoreTest_cellID_reader : public GaudiAlgorithm { public: @@ -35,5 +35,6 @@ class k4FWCoreTest_cellID_reader : public GaudiAlgorithm { /// Handle for the SimTrackerHits to be read DataHandle m_simTrackerHitReaderHandle{"SimTrackerHits", Gaudi::DataHandle::Reader, this}; + MetaDataHandle m_cellIDHandle{m_simTrackerHitReaderHandle, "CellIDEncodingString", Gaudi::DataHandle::Reader, this}; }; #endif /* K4FWCORE_K4FWCORETEST_CELLID */ diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp index afe80de2..929450fb 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.cpp @@ -3,7 +3,7 @@ DECLARE_COMPONENT(k4FWCoreTest_cellID_writer) k4FWCoreTest_cellID_writer::k4FWCoreTest_cellID_writer(const std::string& aName, ISvcLocator* aSvcLoc) - : GaudiAlgorithm(aName, aSvcLoc), m_eventDataSvc("EventDataSvc", "k4FWCoreTest_cellID_writer") { + : GaudiAlgorithm(aName, aSvcLoc) { ; declareProperty("simtrackhits_w", m_simTrackerHitWriterHandle, "Dummy Hit collection Writer"); } @@ -14,15 +14,8 @@ StatusCode k4FWCoreTest_cellID_writer::initialize() { if (GaudiAlgorithm::initialize().isFailure()) { return StatusCode::FAILURE; } - - StatusCode sc = m_eventDataSvc.retrieve(); - m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); - - if (sc == StatusCode::FAILURE) { - error() << "Error retrieving Event Data Service" << endmsg; - return StatusCode::FAILURE; - } - + m_cellIDHandle.put(cellIDtest); + return StatusCode::SUCCESS; } @@ -31,10 +24,6 @@ StatusCode k4FWCoreTest_cellID_writer::execute() { auto hit = simTrackerHits->create(); hit.setPosition({3, 4, 5}); - // TODO: decide on collection metadata interface in the data service - // something along the lines of - // m_podioDataSvc->addCollectionMetaData(collID, "name", value); - return StatusCode::SUCCESS; } diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h index ce87e91b..e159250e 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h @@ -4,9 +4,9 @@ // GAUDI #include "GaudiAlg/GaudiAlgorithm.h" -// edm4hep -#include "TTree.h" +// key4hep #include "k4FWCore/DataHandle.h" +#include "k4FWCore/MetaDataHandle.h" // datamodel #include "edm4hep/SimTrackerHitCollection.h" @@ -34,11 +34,11 @@ class k4FWCoreTest_cellID_writer : public GaudiAlgorithm { virtual StatusCode finalize() final; private: - PodioDataSvc* m_podioDataSvc; - ServiceHandle m_eventDataSvc; /// Handle for the SimTrackerHits to be written DataHandle m_simTrackerHitWriterHandle{"SimTrackerHits", Gaudi::DataHandle::Writer, this}; + MetaDataHandle m_cellIDHandle{m_simTrackerHitWriterHandle, "CellIDEncodingString", Gaudi::DataHandle::Writer, this}; + }; #endif /* K4FWCORE_K4FWCORETEST_CELLID_WRITER */ From 62d9b69227f56ecb87b4fde23d85b019d1c03a69 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 25 May 2023 14:29:09 +0200 Subject: [PATCH 16/24] remove algorithm dependency from MetaDataHandle --- k4FWCore/include/k4FWCore/MetaDataHandle.h | 19 +++++++++---------- .../components/k4FWCoreTest_cellID_reader.h | 2 +- .../components/k4FWCoreTest_cellID_writer.h | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/k4FWCore/include/k4FWCore/MetaDataHandle.h b/k4FWCore/include/k4FWCore/MetaDataHandle.h index 2e4b83be..86790f70 100644 --- a/k4FWCore/include/k4FWCore/MetaDataHandle.h +++ b/k4FWCore/include/k4FWCore/MetaDataHandle.h @@ -14,8 +14,8 @@ template class MetaDataHandle { public: MetaDataHandle(); - MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); - MetaDataHandle(const Gaudi::DataHandle& handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg); + MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a); + MetaDataHandle(const Gaudi::DataHandle& handle, const std::string& descriptor, Gaudi::DataHandle::Mode a); ~MetaDataHandle(); const T get(); @@ -28,7 +28,6 @@ template class MetaDataHandle { ServiceHandle m_eds; std::string m_descriptor; PodioDataSvc* m_podio_data_service{nullptr}; - Algorithm* m_fatherAlg{nullptr}; const Gaudi::DataHandle* m_dataHandle{nullptr}; // holds the identifier in case we do collection metadata Gaudi::DataHandle::Mode m_mode; @@ -39,25 +38,25 @@ template MetaDataHandle::~MetaDataHandle() { //--------------------------------------------------------------------------- template -MetaDataHandle::MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) -: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg), m_mode(a) { +MetaDataHandle::MetaDataHandle(const std::string& descriptor, Gaudi::DataHandle::Mode a) +: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_mode(a) { StatusCode sc = m_eds.retrieve(); m_podio_data_service = dynamic_cast(m_eds.get()); if (nullptr == m_podio_data_service) { - m_fatherAlg->fatal() << "MetaDataHandles require the PodioDataSvc" << endmsg; + std::cout << "ERROR: MetaDataHandles require the PodioDataSvc" << std::endl; } } //--------------------------------------------------------------------------- template -MetaDataHandle::MetaDataHandle(const Gaudi::DataHandle& handle, const std::string& descriptor, Gaudi::DataHandle::Mode a, Algorithm* fatherAlg) -: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_fatherAlg(fatherAlg), m_dataHandle(&handle), m_mode(a) { +MetaDataHandle::MetaDataHandle(const Gaudi::DataHandle& handle, const std::string& descriptor, Gaudi::DataHandle::Mode a) +: m_eds("EventDataSvc", "DataHandle"), m_descriptor(descriptor), m_dataHandle(&handle), m_mode(a) { StatusCode sc = m_eds.retrieve(); m_podio_data_service = dynamic_cast(m_eds.get()); if (nullptr == m_podio_data_service) { - m_fatherAlg->fatal() << "MetaDataHandles require the PodioDataSvc" << endmsg; + std::cout << "ERROR: MetaDataHandles require the PodioDataSvc" << std::endl; } } @@ -76,7 +75,7 @@ template void MetaDataHandle::put(T value) { // check whether we are in the proper State // put is only allowed in the initalization if (m_podio_data_service->targetFSMState() == Gaudi::StateMachine::RUNNING) { - m_fatherAlg->fatal() << "A MetaDataHandle::put cannot be used during the event loop" << endmsg; + throw GaudiException("MetaDataHandle policy violation", "Put cannot be used during the event loop", StatusCode::FAILURE); } std::string full_descriptor = fullDescriptor(); podio::Frame& frame = m_podio_data_service->getMetaDataFrame(); diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h index 48dc6d60..471546e5 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_reader.h @@ -35,6 +35,6 @@ class k4FWCoreTest_cellID_reader : public GaudiAlgorithm { /// Handle for the SimTrackerHits to be read DataHandle m_simTrackerHitReaderHandle{"SimTrackerHits", Gaudi::DataHandle::Reader, this}; - MetaDataHandle m_cellIDHandle{m_simTrackerHitReaderHandle, "CellIDEncodingString", Gaudi::DataHandle::Reader, this}; + MetaDataHandle m_cellIDHandle{m_simTrackerHitReaderHandle, "CellIDEncodingString", Gaudi::DataHandle::Reader}; }; #endif /* K4FWCORE_K4FWCORETEST_CELLID */ diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h index e159250e..4a9a222a 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_cellID_writer.h @@ -38,7 +38,7 @@ class k4FWCoreTest_cellID_writer : public GaudiAlgorithm { /// Handle for the SimTrackerHits to be written DataHandle m_simTrackerHitWriterHandle{"SimTrackerHits", Gaudi::DataHandle::Writer, this}; - MetaDataHandle m_cellIDHandle{m_simTrackerHitWriterHandle, "CellIDEncodingString", Gaudi::DataHandle::Writer, this}; + MetaDataHandle m_cellIDHandle{m_simTrackerHitWriterHandle, "CellIDEncodingString", Gaudi::DataHandle::Writer}; }; #endif /* K4FWCORE_K4FWCORETEST_CELLID_WRITER */ From 8f4a3da810588d59903c30d6b9af84de4885e231 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 25 May 2023 14:50:14 +0200 Subject: [PATCH 17/24] make descriptor generation function publicly available --- k4FWCore/include/k4FWCore/MetaDataHandle.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/k4FWCore/include/k4FWCore/MetaDataHandle.h b/k4FWCore/include/k4FWCore/MetaDataHandle.h index 86790f70..229b7720 100644 --- a/k4FWCore/include/k4FWCore/MetaDataHandle.h +++ b/k4FWCore/include/k4FWCore/MetaDataHandle.h @@ -11,6 +11,10 @@ #include +static std::string metaDataHandleDescriptor(const std::string& collectionName, const std::string& key) { + return collectionName + "__" + key; +} + template class MetaDataHandle { public: MetaDataHandle(); @@ -87,7 +91,7 @@ template std::string MetaDataHandle::fullDescriptor() { std::string full_descriptor; if (nullptr != m_dataHandle) { - full_descriptor = m_dataHandle->objKey() + "__" + m_descriptor; + full_descriptor = metaDataHandleDescriptor(m_dataHandle->objKey(), m_descriptor); // remove the "/Event/" part of the collections' object key if in read mode if (m_mode == Gaudi::DataHandle::Reader) { full_descriptor.erase(0,7); From 166ac859fa447d4bdad1ae28d5c55a1a65818600 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Thu, 25 May 2023 21:15:28 +0200 Subject: [PATCH 18/24] introduce namespace --- k4FWCore/include/k4FWCore/MetaDataHandle.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/k4FWCore/include/k4FWCore/MetaDataHandle.h b/k4FWCore/include/k4FWCore/MetaDataHandle.h index 229b7720..7fcce016 100644 --- a/k4FWCore/include/k4FWCore/MetaDataHandle.h +++ b/k4FWCore/include/k4FWCore/MetaDataHandle.h @@ -11,10 +11,14 @@ #include +namespace k4FWCore { + static std::string metaDataHandleDescriptor(const std::string& collectionName, const std::string& key) { return collectionName + "__" + key; } +} // namespace + template class MetaDataHandle { public: MetaDataHandle(); @@ -91,7 +95,7 @@ template std::string MetaDataHandle::fullDescriptor() { std::string full_descriptor; if (nullptr != m_dataHandle) { - full_descriptor = metaDataHandleDescriptor(m_dataHandle->objKey(), m_descriptor); + full_descriptor = k4FWCore::metaDataHandleDescriptor(m_dataHandle->objKey(), m_descriptor); // remove the "/Event/" part of the collections' object key if in read mode if (m_mode == Gaudi::DataHandle::Reader) { full_descriptor.erase(0,7); From 35e9c8ca8cd106df9b6819daa3ef340b97851ed2 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 2 Jun 2023 12:59:10 +0200 Subject: [PATCH 19/24] Make legacy output string_view safe --- k4FWCore/components/PodioLegacyOutput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4FWCore/components/PodioLegacyOutput.cpp b/k4FWCore/components/PodioLegacyOutput.cpp index 72a11ebc..fcbb5c0f 100644 --- a/k4FWCore/components/PodioLegacyOutput.cpp +++ b/k4FWCore/components/PodioLegacyOutput.cpp @@ -104,7 +104,7 @@ void PodioLegacyOutput::createBranches(const std::vectorgetCollectionIDs()->collectionID(collName); // No check necessary, only registered collections possible auto coll = collNamePair.second; - const auto collType = coll->getValueTypeName() + "Collection"; + const auto collType = std::string(coll->getValueTypeName()) + "Collection"; collectionInfo->emplace_back(collID, std::move(collType), coll->isSubsetCollection()); //} From 544ddbe306e67837e149979d780d9189e7367f47 Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com> Date: Thu, 22 Jun 2023 08:58:45 +0200 Subject: [PATCH 20/24] Update test/k4FWCoreTest/CMakeLists.txt --- test/k4FWCoreTest/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index 1ce3524c..0f166093 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -58,7 +58,8 @@ add_test(NAME CheckExampleEventData ) set_test_env(CheckExampleEventData) set_tests_properties( CheckExampleEventData - PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop") + PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop" + DEPENDS CreateExampleEventData) add_test(NAME CreateExampleEventData_cellID WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} From 478dbb5e7a282e9ae4541a1ef74ed4739efb7c05 Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com> Date: Thu, 22 Jun 2023 08:59:08 +0200 Subject: [PATCH 21/24] Update k4FWCore/src/PodioDataSvc.cpp --- k4FWCore/src/PodioDataSvc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4FWCore/src/PodioDataSvc.cpp b/k4FWCore/src/PodioDataSvc.cpp index 822ee687..989e340e 100644 --- a/k4FWCore/src/PodioDataSvc.cpp +++ b/k4FWCore/src/PodioDataSvc.cpp @@ -33,7 +33,7 @@ StatusCode PodioDataSvc::initialize() { } } - if (m_reading_from_file == true) { + if (m_reading_from_file) { m_metadataframe = m_reader.readEntry("metadata",0); } else { m_metadataframe = podio::Frame(); From 629e628f789187811ed4b2868a688c2673c204bd Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 22 Jun 2023 09:05:04 +0200 Subject: [PATCH 22/24] Remove FWFloatProducer from the README --- README.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7a7e70d6..849fec61 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ -# k4FWCore (Key4HEP FrameWork Core) - - -k4FWCore is a Gaudi package that provides the PodioDataService, that allows to use podio-based event data models like edm4hep in gaudi workflows. +# k4FWCore (key4hep FrameWork Core) +k4FWCore is a Gaudi package that provides the PodioDataService, that allows to +use podio-based event data models like EDM4hep in Gaudi workflows. ## Components @@ -12,7 +11,6 @@ k4FWCore is a Gaudi package that provides the PodioDataService, that allows to Component wrapping the PodioDataService to handle PODIO types and collections. - #### PodioInput Algorithm to read data from input file(s) on disk. @@ -21,10 +19,6 @@ Algorithm to read data from input file(s) on disk. Algorithm to write data to output file on disk. -#### FWFloatProducer - -Algorithm with a simple float as output, for testing and adding parameters to jobs. - ## Dependencies From e8716318194b3080902df5a507c686824e4f9588 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 22 Jun 2023 09:08:56 +0200 Subject: [PATCH 23/24] Remove unneeded TFile include --- k4FWCore/components/PodioOutput.cpp | 1 - k4FWCore/components/PodioOutput.h | 1 - 2 files changed, 2 deletions(-) diff --git a/k4FWCore/components/PodioOutput.cpp b/k4FWCore/components/PodioOutput.cpp index 2b3ac519..9c976e56 100644 --- a/k4FWCore/components/PodioOutput.cpp +++ b/k4FWCore/components/PodioOutput.cpp @@ -1,7 +1,6 @@ #include #include "PodioOutput.h" -#include "TFile.h" #include "k4FWCore/PodioDataSvc.h" #include "rootUtils.h" diff --git a/k4FWCore/components/PodioOutput.h b/k4FWCore/components/PodioOutput.h index 6327ff74..222ae4a1 100644 --- a/k4FWCore/components/PodioOutput.h +++ b/k4FWCore/components/PodioOutput.h @@ -9,7 +9,6 @@ #include // forward declarations -class TFile; class PodioDataSvc; class PodioOutput : public GaudiAlgorithm { From e3fb057a6a0678040b45b30d41f890ef822fbacd Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Fri, 23 Jun 2023 09:07:25 +0200 Subject: [PATCH 24/24] silence MetaDataHandle warnings during genconf step --- k4FWCore/include/k4FWCore/MetaDataHandle.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/k4FWCore/include/k4FWCore/MetaDataHandle.h b/k4FWCore/include/k4FWCore/MetaDataHandle.h index 7fcce016..05fb9f93 100644 --- a/k4FWCore/include/k4FWCore/MetaDataHandle.h +++ b/k4FWCore/include/k4FWCore/MetaDataHandle.h @@ -31,6 +31,7 @@ template class MetaDataHandle { private: std::string fullDescriptor(); + void checkPodioDataSvc(); private: ServiceHandle m_eds; @@ -51,9 +52,7 @@ MetaDataHandle::MetaDataHandle(const std::string& descriptor, Gaudi::DataHand StatusCode sc = m_eds.retrieve(); m_podio_data_service = dynamic_cast(m_eds.get()); - if (nullptr == m_podio_data_service) { - std::cout << "ERROR: MetaDataHandles require the PodioDataSvc" << std::endl; - } + checkPodioDataSvc(); } //--------------------------------------------------------------------------- @@ -63,9 +62,7 @@ MetaDataHandle::MetaDataHandle(const Gaudi::DataHandle& handle, const std::st StatusCode sc = m_eds.retrieve(); m_podio_data_service = dynamic_cast(m_eds.get()); - if (nullptr == m_podio_data_service) { - std::cout << "ERROR: MetaDataHandles require the PodioDataSvc" << std::endl; - } + checkPodioDataSvc(); } @@ -105,4 +102,16 @@ template std::string MetaDataHandle::fullDescriptor() { } return full_descriptor; } + +//--------------------------------------------------------------------------- +template void MetaDataHandle::checkPodioDataSvc(){ + // do not do this check during the genconf step + const std::string cmd = System::cmdLineArgs()[0]; + if ( cmd.find( "genconf" ) != std::string::npos ) return; + + if (nullptr == m_podio_data_service) { + std::cout << "ERROR: MetaDataHandles require the PodioDataSvc" << std::endl; + } +} + #endif