Add support for interface types (#516)

* Make code generation process interfaces

* Implement interface types using type-erasure

* Add unittests for interface types

* Make sure to always store the immutable handle inside

* Put all type helpers into the podio namespace

Avoid collisions downstream

* make equality operators symmetric

* Return trivial types by value for consistency

* Improve method name to be more interface like

* Make interfaces only accept interfaced types

* Fix easy to fix python issues

* Add documentation for interface types

* Make checking for explicit types and if-else chain

cmake/podioMacros.cmake

@@ -203,8 +203,11 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR
     ${YAML_FILE}
     ${PODIO_TEMPLATES}
     ${podio_PYTHON_DIR}/podio_class_generator.py
-    ${podio_PYTHON_DIR}/podio/generator_utils.py
-    ${podio_PYTHON_DIR}/podio/podio_config_reader.py
+    ${podio_PYTHON_DIR}/podio_gen/generator_utils.py
+    ${podio_PYTHON_DIR}/podio_gen/podio_config_reader.py
+    ${podio_PYTHON_DIR}/podio_gen/generator_base.py
+    ${podio_PYTHON_DIR}/podio_gen/cpp_generator.py
+    ${podio_PYTHON_DIR}/podio_gen/julia_generator.py
   )
 
   message(STATUS "Creating '${datamodel}' datamodel")

doc/datamodel_syntax.md

@@ -9,6 +9,7 @@ PODIO encourages the usage of composition, rather than inheritance.
 One of the reasons for doing so is the focus on efficiency friendly `plain-old-data`.
 For this reason, PODIO does not support inheritance within the defined data model.
 Instead, users can combine multiple `components` to build a to be used `datatype`.
+Additionally, if several datatypes should be callable through the same interface, the `interface` category can be used.
 
 To allow the datatypes to be real PODs, the data stored within the data model are constrained to be
 POD-compatible data. Those are
@@ -92,7 +93,6 @@ For describing physics quantities it is important to know their units. Thus it i
       <type> <name>{<default-value>} [<unit>] // <comment>
 ```
 
-
 ### Definition of references between objects:
 There can be one-to-one-relations and one-to-many relations being stored in a particular class. This happens either in the `OneToOneRelations` or `OneToManyRelations` section of the data definition. The definition has again the form:
 
@@ -126,6 +126,57 @@ The `includes` will be add to the header files of the generated classes.
 
 The code being provided has to use the macro `{name}` in place of the concrete name of the class.
 
+## Definition of custom interfaces
+An interface type can be defined as follows (again using an example from the example datamodel)
+
+```yaml
+    interfaces:
+      TypeWithEnergy:
+        Description: "A generic interface for types with an energy member"
+        Author: "Someone"
+        Types:
+          - ExampleHit
+          - ExampleMC
+          - ExampleCluster
+        Members:
+          - double energy // the energy
+```
+
+This definition will yield a class called `TypeWithEnergy` that has one public
+method to get the `energy`. Here the syntax for defining more accessors is
+exactly the same as it is for the `datatypes`. Additionally, it is necessary to
+define which `Types` can be used with this interface class, in this case the
+`ExampleHit`, `ExampleMC` or an `ExampleCluster`. **NOTE: `interface` types do
+not allow for mutable access to their data.** They can be used in relations
+between objects, just like normal `datatypes`.
+
+### Assigning to interface types
+
+Interface types support the same functionality as normal (immutable) datatypes.
+The main additional functionality they offer is the possibility to directly
+assign to them (if they type is supported) and to query them for the type that
+is currently being held internally.
+
+```cpp
+auto energyType = TypeWithEnergy{}; // An empty interface is possible but useless
+bool valid = energyType.isValid();  // <-- false
+
+auto hit = ExampleHit{};
+energyType = hit;     // assigning a hit to the interface type
+energyType.energy();  // <-- get's the energy from the underlying hit
+
+auto cluster = ExampleCluster{};
+energyType = cluster; // ra-assigning another object is possible
+bool same = (energyType == cluster); // <-- true (comparisons work as expected)
+
+bool isCluster = energyType.isA<ExampleCluster>(); // <-- true
+bool isHit     = energyType.isA<ExampleHit>();     // <-- false
+
+auto newCluster = energyType.getValue<ExampleCluster>(); // <-- "cast back" to original type
+
+// "Casting" only works if the types match. Otherwise there will be an exception
+auto newHit = energyType.getValue<ExampleHit>(); // <-- exception
+```
 
 ## Global options
 Some customization of the generated code is possible through flags. These flags are listed in the section `options`:

include/podio/utilities/TypeHelpers.h

@@ -9,6 +9,45 @@
 #include <vector>
 
 namespace podio {
+#if __has_include("experimental/type_traits.h")
+  #include <experimental/type_traits>
+namespace det {
+  using namespace std::experimental;
+} // namespace det
+#else
+// Implement the minimal feature set we need
+namespace det {
+  namespace detail {
+    template <typename DefT, typename AlwaysVoidT, template <typename...> typename Op, typename... Args>
+    struct detector {
+      using value_t = std::false_type;
+      using type = DefT;
+    };
+
+    template <typename DefT, template <typename...> typename Op, typename... Args>
+    struct detector<DefT, std::void_t<Op<Args...>>, Op, Args...> {
+      using value_t = std::true_type;
+      using type = Op<Args...>;
+    };
+  } // namespace detail
+
+  struct nonesuch {
+    ~nonesuch() = delete;
+    nonesuch(const nonesuch&) = delete;
+    void operator=(const nonesuch&) = delete;
+  };
+
+  template <template <typename...> typename Op, typename... Args>
+  using is_detected = typename detail::detector<nonesuch, void, Op, Args...>::value_t;
+
+  template <template <typename...> typename Op, typename... Args>
+  static constexpr bool is_detected_v = is_detected<Op, Args...>::value;
+
+  template <typename DefT, template <typename...> typename Op, typename... Args>
+  using detected_or = detail::detector<DefT, void, Op, Args...>;
+} // namespace det
+#endif
+
 namespace detail {
 
   /**
@@ -158,6 +197,60 @@ namespace detail {
   template <typename T>
   using GetMappedType = typename MapLikeTypeHelper<T>::mapped_type;
 
+  /**
+   * Detector for checking the existence of a mutable_type type member. Used to
+   * determine whether T is (or could be) a podio generated default (immutable)
+   * handle.
+   */
+  template <typename T>
+  using hasMutable_t = typename T::mutable_type;
+
+  /**
+   * Detector for checking the existance of an object_type type member. Used ot
+   * determine whether T is (or could be) a podio generated mutable handle.
+   */
+  template <typename T>
+  using hasObject_t = typename T::object_type;
+
+  /**
+   * Variable template for determining whether type T is a podio generated
+   * handle class.
+   *
+   * NOTE: this basically just checks the existance of the mutable_type and
+   * object_type type members, so it is rather easy to fool this check if one
+   * wanted to. However, for our purposes we mainly need it for a static_assert
+   * to have more understandable compilation error message.
+   */
+  template <typename T>
+  constexpr static bool isPodioType = det::is_detected_v<hasObject_t, T> || det::is_detected_v<hasMutable_t, T>;
+
+  /**
+   * Variable template for obtaining the default handle type from any podio
+   * generated handle type.
+   *
+   * If T is already a default handle, this will return T, if T is a mutable
+   * handle it will return T::object_type.
+   */
+  template <typename T>
+  using GetDefaultHandleType = typename det::detected_or<T, hasObject_t, T>::type;
+
+  /**
+   * Variable template for obtaining the mutable handle type from any podio
+   * generated handle type.
+   *
+   * If T is alrady a mutable handle, this will return T, if T is a default
+   * handle it will return T::mutable_type.
+   */
+  template <typename T>
+  using GetMutableHandleType = typename det::detected_or<T, hasMutable_t, T>::type;
+
+  /**
+   * Helper type alias to transform a tuple of handle types to a tuple of
+   * mutable handle types.
+   */
+  template <typename Tuple>
+  using TupleOfMutableTypes = typename ToTupleOfTemplateHelper<GetMutableHandleType, Tuple>::type;
+
 } // namespace detail
 
 // forward declaration to be able to use it below

python/podio_gen/cpp_generator.py

@@ -103,6 +103,8 @@ def do_process_component(self, name, component):
       self._fill_templates('Component', component)
       self.root_schema_component_names.add(name + self.old_schema_version)
 
+    return component
+
   def do_process_datatype(self, name, datatype):
     """Do the cpp specific processing of a datatype"""
     datatype["includes_data"] = self._get_member_includes(datatype["Members"])
@@ -152,6 +154,17 @@ def do_process_datatype(self, name, datatype):
     if 'SIO' in self.io_handlers:
       self._fill_templates('SIOBlock', datatype)
 
+    return datatype
+
+  def do_process_interface(self, _, interface):
+    """Process an interface definition and generate the necesary code"""
+    interface["include_types"] = [
+        self._build_include(t) for t in interface["Types"]
+        ]
+
+    self._fill_templates("Interface", interface)
+    return interface
+
   def print_report(self):
     """Print a summary report about the generated code"""
     if not self.verbose:
@@ -176,6 +189,8 @@ def _preprocess_for_class(self, datatype):
         member.sub_members = self.datamodel.components[member.full_type]['Members']
 
     for relation in datatype['OneToOneRelations']:
+      if self._is_interface(relation.full_type):
+        relation.interface_types = self.datamodel.interfaces[relation.full_type]["Types"]
       if self._needs_include(relation.full_type):
         if relation.namespace not in fwd_declarations:
           fwd_declarations[relation.namespace] = []
@@ -188,6 +203,8 @@ def _preprocess_for_class(self, datatype):
       includes.add('#include "podio/RelationRange.h"')
 
     for relation in datatype['OneToManyRelations']:
+      if self._is_interface(relation.full_type):
+        relation.interface_types = self.datamodel.interfaces[relation.full_type]["Types"]
       if self._needs_include(relation.full_type):
         includes.add(self._build_include(relation))
 
@@ -243,7 +260,12 @@ def _preprocess_for_collection(self, datatype):
     for relation in datatype['OneToManyRelations'] + datatype['OneToOneRelations']:
       if datatype['class'].bare_type != relation.bare_type:
         include_from = self._needs_include(relation.full_type)
-        includes_cc.add(self._build_include_for_class(relation.bare_type + 'Collection', include_from))
+        if self._is_interface(relation.full_type):
+          includes_cc.add(self._build_include_for_class(relation.bare_type, include_from))
+          for int_type in relation.interface_types:
+            includes_cc.add(self._build_include_for_class(int_type.bare_type + 'Collection', include_from))
+        else:
+          includes_cc.add(self._build_include_for_class(relation.bare_type + 'Collection', include_from))
         includes.add(self._build_include_for_class(relation.bare_type, include_from))
 
     if datatype['VectorMembers']:
@@ -425,11 +447,15 @@ def _get_member_includes(self, members):
 
   def _needs_include(self, classname) -> IncludeFrom:
     """Check whether the member needs an include from within the datamodel"""
-    if classname in self.datamodel.components or classname in self.datamodel.datatypes:
+    if classname in self.datamodel.components or \
+       classname in self.datamodel.datatypes or \
+       classname in self.datamodel.interfaces:
       return IncludeFrom.INTERNAL
 
     if self.upstream_edm:
-      if classname in self.upstream_edm.components or classname in self.upstream_edm.datatypes:
+      if classname in self.upstream_edm.components or \
+         classname in self.upstream_edm.datatypes or \
+         classname in self.upstream_edm.interfaces:
         return IncludeFrom.EXTERNAL
 
     return IncludeFrom.NOWHERE

python/podio_gen/generator_base.py

@@ -73,26 +73,38 @@ class ClassGeneratorBaseMixin:
                          processed. Needs to return a (potentially) empty
                          dictionary
 
-  do_process_component(name: str, component: dict): do some language specific
-                       processing for a component populating the component
-                       dictionary further. When called only the "class" key will
-                       be populated. This function also has to to take care of
-                       filling the necessary templates!
+  do_process_component(name: str, component: dict) -> dict: do some language
+                       specific processing for a component populating the
+                       component dictionary further. When called only the
+                       "class" key will be populated. Return a dictionary or
+                       None. If None, this will not be put into the "components"
+                       list. This function also has to to take care of filling
+                       the necessary templates!
 
   do_process_datatype(name: str, datatype: dict): do some language specific
                       processing for a datatype populating the datatype
                       dictionary further. When called only the "class" key will
-                      be populated. This function also has to take care of
-                      filling the necessary templates!
+                      be populated. Return a dictionary or None. If None, this
+                      will not be put into the "datatypes" list. This function
+                      also has to take care of filling the necessary templates!
+
+  do_process_interface(name: str, interface: dict): do some language specific
+                       processing for an interface type, populating the
+                       interface dictionary further. When called only the
+                       "class" key will be populated. Return a dictionary or
+                       None. If None, this will not be put into the "interfaces"
+                       list. This function also has to take care of filling the
+                       necessary templates!
 
   post_process(datamodel: dict): do some global post processing for which all
                components and datatypes need to have been processed already.
                Gets called with the dictionary that has been created in
                pre_proces and filled during the processing. The process
-               components and datatypes are accessible via the "components" and
-               "datatypes" keys respectively.
+               components and datatypes are accessible via the "components",
+               "datatypes" and "interfaces" keys respectively.
 
   print_report(): prints a report summarizing what has been generated
+
   """
   def __init__(self, yamlfile, install_dir, package_name, verbose, dryrun, upstream_edm):
     self.yamlfile = yamlfile
@@ -128,12 +140,22 @@ def process(self):
 
     datamodel['components'] = []
     datamodel['datatypes'] = []
+    datamodel['interfaces'] = []
 
     for name, component in self.datamodel.components.items():
-      datamodel["components"].append(self._process_component(name, component))
+      comp = self._process_component(name, component)
+      if comp is not None:
+        datamodel["components"].append(comp)
 
     for name, datatype in self.datamodel.datatypes.items():
-      datamodel["datatypes"].append(self._process_datatype(name, datatype))
+      datat = self._process_datatype(name, datatype)
+      if datat is not None:
+        datamodel["datatypes"].append(datat)
+
+    for name, interface in self.datamodel.interfaces.items():
+      interf = self._process_interface(name, interface)
+      if interf is not None:
+        datamodel["interfaces"].append(interf)
 
     self.post_process(datamodel)
     if self.verbose:
@@ -147,18 +169,23 @@ def _process_component(self, name, component):
     component = deepcopy(component)
     component['class'] = DataType(name)
 
-    self.do_process_component(name, component)
-    return component
+    return self.do_process_component(name, component)
 
   def _process_datatype(self, name, datatype):
     """Process a single datatype into a dictionary that can be used in jinja2
     templates and return that"""
     datatype = deepcopy(datatype)
     datatype["class"] = DataType(name)
 
-    self.do_process_datatype(name, datatype)
+    return self.do_process_datatype(name, datatype)
 
-    return datatype
+  def _process_interface(self, name, interface):
+    """Process a single interface definition into a dictionary that can be used
+    in jinja2 templates and return that"""
+    interface = deepcopy(interface)
+    interface["class"] = DataType(name)
+
+    return self.do_process_interface(name, interface)
 
   @staticmethod
   def _get_filenames_templates(template_base, name):
@@ -182,6 +209,7 @@ def get_fn_format(tmpl):
     endings = {
         'Data': ('h',),
         'PrintInfo': ('h',),
+        'Interface': ('h',),
         'MutableStruct': ('jl',),
         'ParentModule': ('jl',),
         }.get(template_base, ('h', 'cc'))
@@ -228,3 +256,10 @@ def _fill_templates(self, template_base, data, old_schema_data=None):
     data['incfolder'] = self.incfolder
     for filename, template in self._get_filenames_templates(template_base, data['class'].bare_type):
       self._write_file(filename, self._eval_template(template, data, old_schema_data))
+
+  def _is_interface(self, classname):
+    """Check whether this is an interface type or a regular datatype"""
+    all_interfaces = self.datamodel.interfaces
+    if self.upstream_edm:
+      all_interfaces = list(self.datamodel.interfaces) + list(self.upstream_edm.interfaces)
+    return classname in all_interfaces