Sort loaded plugins based on their dependencies (implements #894)

* Refactor the 'module list dependency sort' function into
  utils.sort.dependency
* Use the dependency sort for modules and plugins

netsim/augment/plugin.py

@@ -9,8 +9,8 @@
 import importlib.util
 
 from box import Box
-from ..utils import log, read as _read
-from ..utils.files import get_moddir
+from ..utils import log, read as _read, sort as _sort
+from ..utils.files import get_moddir,get_search_path
 from .. import data
 from . import config
 
@@ -89,6 +89,30 @@ def load_plugin_from_path(path: str, plugin: str, topology: Box) -> typing.Optio
     print(f"loaded plugin {module_name}")
   return pymodule
 
+'''
+Sort plugins based on their _requires and _execute_after attributes
+
+Input:
+* List of plugins in topology.plugin
+* Loaded plugins (in the same order) in topology.Plugin
+
+The sorting function has to build a dictionary of plugin modules, sort the plugin names,
+and rebuild the list of loaded plugins.
+'''
+def sort_plugins(topology: Box) -> None:
+  if not topology.get('plugin',[]):                           # No plugins, no sorting ;)
+    return
+
+  pmap: dict = {}
+  for (idx,p) in enumerate(topology.plugin):                  # Build the name-to-module mappings
+    pmap[p] = topology.Plugin[idx]
+
+  # Sort the plugin names based on their dependencies
+  topology.plugin = _sort.dependency(
+                      topology.plugin,
+                      lambda p: getattr(pmap[p],'_requires',[]) + getattr(pmap[p],'_execute_after',[]))
+  topology.Plugin = [ pmap[p] for p in topology.plugin ]      # And rebuild the list of plugin modules
+
 def init(topology: Box) -> None:
   data.types.must_be_list(parent=topology,key='defaults.plugin',path='',create_empty=True)    # defaults.plugin must be a list (if present)
   if topology.defaults.plugin:                                                                # If we have default plugins...
@@ -99,8 +123,8 @@ def init(topology: Box) -> None:
     return
 
   topology.Plugin = []
+  search_path = get_search_path(pkg_path_component='extra')   # Search the usual places plus the 'extra' package directory
   for pname in list(topology.plugin):                         # Iterate over all plugins
-    search_path = ('.',str(get_moddir() / 'extra'))           # Search in current directory and 'extra' package directory
     for path in search_path:
       plugin = load_plugin_from_path(path,pname,topology)     # Try to load plugin from the current search path directory
       if plugin:                                              # Got it, get out of the loop
@@ -111,6 +135,7 @@ def init(topology: Box) -> None:
     else:
       log.error(f"Cannot find plugin {pname}\nSearch path: {search_path}",log.IncorrectValue,'plugin')
 
+  sort_plugins(topology)
   if log.debug_active('plugin'):
     print(f'plug INIT: {topology.Plugin}')
 

netsim/extra/ebgp.multihop/plugin.py

@@ -5,6 +5,9 @@
 from netsim.augment import links
 from netsim.data.validate import validate_attributes
 
+_config_name = 'ebgp.multihop'
+_execute_after = [ 'ebgp.utils', 'bgp.session' ]
+
 def pre_transform(topology: Box) -> None:
   config_name  = api.get_config_name(globals())        # Get the plugin configuration name
   session_idx  = data.find_in_list(['ebgp.utils','bgp.session'],topology.plugin)

netsim/modules/__init__.py

@@ -10,7 +10,7 @@
 
 # Related modules
 from .. import data
-from ..utils import log
+from ..utils import log,sort as _sort
 from ..data.validate import must_be_list
 from ..utils.callback import Callback
 from ..augment import devices
@@ -433,23 +433,8 @@ def reorder_node_modules(topology: Box, secondary_sort: str = "config_after") ->
       n.module = sort_module_list(n.module,topology.defaults, secondary_sort)
 
 def sort_module_list(mods: list, mod_params: Box, secondary_sort: str = "config_after") -> list:
-  if (len(mods) < 2):
-    return mods
-
-  output: typing.List[str] = []
-  while len(mods):
-    skipped: typing.List[str] = []
-    for m in mods:
-      if m in mod_params:
-        requires = mod_params[m].get('requires',[]) + mod_params[m].get(secondary_sort,[])
-        if [ r for r in requires if r in mods ]:
-          skipped = skipped + [ m ]
-        else:
-          output = output + [ m ]
-
-    mods = skipped
-
-  return output
+  mods = [ m for m in mods if m in mod_params ]
+  return _sort.dependency(mods,lambda m: mod_params[m].get('requires',[]) + mod_params[m].get(secondary_sort,[]))
 
 """
 Copy node data into interface data:

netsim/utils/sort.py

@@ -0,0 +1,43 @@
+# Implement simple sorting routines
+
+import typing
+from . import log
+
+'''
+Sort objects based on their dependencies
+
+Given a name of an object, the 'get_dependency' callable should return
+the list of object names this object depends on.
+
+The function tries to sort object based on their dependencies using a very
+simple algorithm that iterates over the list of remaining objects as many
+times as needed (with the initial length of the list being a safeguard)
+
+* Take the next object from the list
+* Is there another object in the remaining objects list that this one
+  depends on? If so skip it, otherwise it's safe to add the object
+  to the sorted list.
+'''
+
+def dependency(mods: list, get_dependency: typing.Callable[[str],list]) -> list:
+  if (len(mods) < 2):
+    return mods
+
+  watchdog_counter = len(mods)
+
+  output: typing.List[str] = []
+  while len(mods):
+    skipped: typing.List[str] = []
+    for m in mods:
+      requires = get_dependency(m)
+      if [ r for r in requires if r in mods ]:
+        skipped = skipped + [ m ]
+      else:
+        output = output + [ m ]
+
+    mods = skipped
+    watchdog_counter -= 1
+    if (watchdog_counter < 0):
+      raise log.FatalError('dependency sort encountered a loop')
+
+  return output