Refactor Edalizer class

Refactor this class to split out functionality into smaller
sub-functions to make it easier to extend. In the process, remove some
Python 2 functionality we don't need any more. The call to the depenceny
resolver moved into the Edalize class as well to provide a slightly
cleaner interface: a CoreManager instance, and the top-level core.

Updated the tests to fulfill the new requirement to pass a CoreManager
instance to Edalizer. Move some test files into separate directories to
keep core libraries separate and avoid spilling between tests.

Thanks to Rupert for working on the tests.

Co-authored-by: Rupert Swarbrick <[email protected]>

fusesoc/edalizer.py

@@ -6,6 +6,7 @@
 from fusesoc import utils
 from fusesoc.vlnv import Vlnv
 from fusesoc.utils import merge_dict
+from fusesoc.coremanager import DependencyError
 
 logger = logging.getLogger(__name__)
 
@@ -21,59 +22,148 @@ def __call__(self, parser, namespace, values, option_string=None):
 class Edalizer:
     def __init__(
         self,
-        vlnv,
+        toplevel,
         flags,
-        cores,
         cache_root,
         work_root,
+        core_manager,
         export_root=None,
         system_name=None,
     ):
-        if os.path.exists(work_root):
-            for f in os.listdir(work_root):
-                if os.path.isdir(os.path.join(work_root, f)):
-                    shutil.rmtree(os.path.join(work_root, f))
+        logger.debug("Building EDA API")
+
+        self.toplevel = toplevel
+        self.flags = flags
+        self.cache_root = cache_root
+        self.core_manager = core_manager
+        self.work_root = work_root
+        self.export_root = export_root
+        self.system_name = system_name
+
+        self._prepare_work_root()
+
+        self.generators = {}
+
+        self._generated_cores = []
+
+    @property
+    def cores(self):
+        return self.resolved_cores + self._generated_cores
+
+    @property
+    def resolved_cores(self):
+        """ Get a list of all "used" cores after the dependency resolution """
+        try:
+            return self.core_manager.get_depends(self.toplevel, self.flags)
+        except DependencyError as e:
+            logger.error(
+                e.msg + "\nFailed to resolve dependencies for {}".format(self.toplevel)
+            )
+            exit(1)
+        except SyntaxError as e:
+            logger.error(e.msg)
+            exit(1)
+
+    @property
+    def discovered_cores(self):
+        """ Get a list of all cores found by fusesoc """
+        return self.core_manager.db.find()
+
+    def run(self):
+        """ Run all steps to create a EDA API YAML file """
+
+        # Run the setup task on all cores (fetch and patch them as needed)
+        self.setup_cores()
+
+        # Get all generators defined in any of the cores
+        self.extract_generators()
+
+        # Run all generators. Generators can create new cores, which are added
+        # to the list of available cores.
+        self.run_generators()
+
+        # Create EDA API file contents
+        self.create_eda_api_struct()
+
+    def _core_flags(self, core):
+        """ Get flags for a specific core """
+
+        core_flags = self.flags.copy()
+        core_flags["is_toplevel"] = core.name == self.toplevel
+        return core_flags
+
+    def _prepare_work_root(self):
+        if os.path.exists(self.work_root):
+            for f in os.listdir(self.work_root):
+                if os.path.isdir(os.path.join(self.work_root, f)):
+                    shutil.rmtree(os.path.join(self.work_root, f))
                 else:
-                    os.remove(os.path.join(work_root, f))
+                    os.remove(os.path.join(self.work_root, f))
         else:
-            os.makedirs(work_root)
+            os.makedirs(self.work_root)
 
-        logger.debug("Building EDA API")
+    def setup_cores(self):
+        """ Setup cores: fetch resources, patch them, etc. """
+        for core in self.cores:
+            logger.info("Preparing " + str(core.name))
+            core.setup()
 
+    def extract_generators(self):
+        """ Get all registered generators from the cores """
         generators = {}
+        for core in self.cores:
+            logger.debug("Searching for generators in " + str(core.name))
+            core_flags = self._core_flags(core)
+
+            if hasattr(core, "get_generators"):
+                core_generators = core.get_generators(core_flags)
+                logger.debug("Found generators: %s" % (core_generators,))
+                generators.update(core_generators)
 
+        self.generators = generators
+
+    def run_generators(self):
+        """ Run all generators """
+        for core in self.cores:
+            logger.debug("Running generators in " + str(core.name))
+            core_flags = self._core_flags(core)
+
+            if hasattr(core, "get_ttptttg"):
+                for ttptttg_data in core.get_ttptttg(core_flags):
+                    _ttptttg = Ttptttg(ttptttg_data, core, self.generators,)
+                    for gen_core in _ttptttg.generate(self.cache_root):
+                        gen_core.pos = _ttptttg.pos
+                        self._generated_cores.append(gen_core)
+
+    def create_eda_api_struct(self):
         first_snippets = []
         snippets = []
         last_snippets = []
-        _flags = flags.copy()
-        core_queue = cores[:]
-        core_queue.reverse()
-        while core_queue:
+        for core in self.cores:
             snippet = {}
-            core = core_queue.pop()
-            logger.info("Preparing " + str(core.name))
-            core.setup()
 
             logger.debug("Collecting EDA API parameters from {}".format(str(core.name)))
-            _flags["is_toplevel"] = core.name == vlnv
+            _flags = self._core_flags(core)
 
             # Extract direct dependencies
             snippet["dependencies"] = {str(core.name): core.direct_deps}
 
             # Extract files
-            if export_root:
-                files_root = os.path.join(export_root, core.sanitized_name)
+            if self.export_root:
+                files_root = os.path.join(self.export_root, core.sanitized_name)
                 core.export(files_root, _flags)
             else:
                 files_root = core.files_root
 
-            rel_root = os.path.relpath(files_root, work_root)
+            rel_root = os.path.relpath(files_root, self.work_root)
 
             # Extract parameters
             snippet["parameters"] = core.get_parameters(_flags)
 
             # Extract tool options
-            snippet["tool_options"] = {flags["tool"]: core.get_tool_options(_flags)}
+            snippet["tool_options"] = {
+                self.flags["tool"]: core.get_tool_options(_flags)
+            }
 
             # Extract scripts
             snippet["hooks"] = core.get_scripts(rel_root, _flags)
@@ -82,7 +172,7 @@ def __init__(
             for file in core.get_files(_flags):
                 if file.copyto:
                     _name = file.copyto
-                    dst = os.path.join(work_root, _name)
+                    dst = os.path.join(self.work_root, _name)
                     _dstdir = os.path.dirname(dst)
                     if not os.path.exists(_dstdir):
                         os.makedirs(_dstdir)
@@ -117,18 +207,6 @@ def __init__(
                     }
                 )
 
-            # Extract generators if defined in CAPI
-            if hasattr(core, "get_generators"):
-                generators.update(core.get_generators(_flags))
-
-            # Run generators
-            if hasattr(core, "get_ttptttg"):
-                for ttptttg_data in core.get_ttptttg(_flags):
-                    _ttptttg = Ttptttg(ttptttg_data, core, generators)
-                    for gen_core in _ttptttg.generate(cache_root):
-                        gen_core.pos = _ttptttg.pos
-                        core_queue.append(gen_core)
-
             if hasattr(core, "pos"):
                 if core.pos == "first":
                     first_snippets.append(snippet)
@@ -139,16 +217,16 @@ def __init__(
             else:
                 snippets.append(snippet)
 
-        top_core = cores[-1]
+        top_core = self.resolved_cores[-1]
         self.edalize = {
             "version": "0.2.1",
             "dependencies": {},
             "files": [],
             "hooks": {},
-            "name": system_name or top_core.sanitized_name,
+            "name": self.system_name or top_core.sanitized_name,
             "parameters": {},
             "tool_options": {},
-            "toplevel": top_core.get_toplevel(flags),
+            "toplevel": top_core.get_toplevel(self.flags),
             "vpi": [],
         }
 

fusesoc/main.py

@@ -409,26 +409,17 @@ def run_backend(
         do_configure = True
 
     if do_configure:
-        try:
-            cores = cm.get_depends(core.name, flags)
-        except DependencyError as e:
-            logger.error(
-                e.msg + "\nFailed to resolve dependencies for {}".format(system)
-            )
-            exit(1)
-        except SyntaxError as e:
-            logger.error(e.msg)
-            exit(1)
         try:
             edalizer = Edalizer(
-                core.name,
-                flags,
-                cores,
+                toplevel=core.name,
+                flags=flags,
+                core_manager=cm,
                 cache_root=cm.config.cache_root,
                 work_root=work_root,
                 export_root=export_root,
                 system_name=system_name,
             )
+            edalizer.run()
 
             backend_class = get_edatool(tool)
             edalizer.parse_args(backend_class, backendargs)

tests/test_capi2.py

@@ -186,7 +186,7 @@ def test_capi2_get_files():
 def test_capi2_get_generators():
     from fusesoc.core import Core
 
-    core = Core(os.path.join(cores_dir, "generators.core"))
+    core = Core(os.path.join(cores_dir, "generate", "generators.core"))
 
     generators = core.get_generators({})
     assert len(generators) == 1
@@ -441,7 +441,7 @@ def test_capi2_get_toplevel():
 def test_capi2_get_ttptttg():
     from fusesoc.core import Core
 
-    core = Core(os.path.join(cores_dir, "generate.core"))
+    core = Core(os.path.join(cores_dir, "generate", "generate.core"))
 
     flags = {"is_toplevel": True}
     expected = [
@@ -517,8 +517,12 @@ def test_capi2_get_work_root():
 def test_capi2_info():
     from fusesoc.core import Core
 
-    for core_name in ["targets", "generators"]:
-        core_file = os.path.join(tests_dir, "capi2_cores", "misc", core_name + ".core")
+    dirs_and_corenames = [("", "targets"), ("generate", "generators")]
+
+    for subdir, core_name in dirs_and_corenames:
+        core_file = os.path.join(
+            tests_dir, "capi2_cores", "misc", subdir, core_name + ".core"
+        )
         core = Core(core_file)
 
         gen_info = "\n".join(

tests/test_coremanager.py

@@ -52,17 +52,36 @@ def test_copyto():
     import os
     import tempfile
 
+    from fusesoc.config import Config
+    from fusesoc.coremanager import CoreManager
     from fusesoc.edalizer import Edalizer
-    from fusesoc.core import Core
+    from fusesoc.librarymanager import Library
+    from fusesoc.vlnv import Vlnv
 
-    core = Core(
-        os.path.join(os.path.dirname(__file__), "cores", "misc", "copytocore.core")
-    )
     flags = {"tool": "icarus"}
 
     work_root = tempfile.mkdtemp(prefix="copyto_")
 
-    eda_api = Edalizer(core.name, flags, [core], None, work_root, None).edalize
+    core_dir = os.path.join(os.path.dirname(__file__), "cores", "misc", "copytocore")
+    lib = Library("misc", core_dir)
+
+    cm = CoreManager(Config())
+    cm.add_library(lib)
+
+    core = cm.get_core(Vlnv("::copytocore"))
+
+    edalizer = Edalizer(
+        toplevel=core.name,
+        flags=flags,
+        core_manager=cm,
+        cache_root=None,
+        work_root=work_root,
+        export_root=None,
+        system_name=None,
+    )
+    edalizer.run()
+
+    eda_api = edalizer.edalize
 
     assert eda_api["files"] == [
         {
@@ -88,25 +107,35 @@ def test_export():
     import os
     import tempfile
 
+    from fusesoc.config import Config
+    from fusesoc.coremanager import CoreManager
     from fusesoc.edalizer import Edalizer
-    from fusesoc.core import Core
+    from fusesoc.librarymanager import Library
+    from fusesoc.vlnv import Vlnv
 
-    core = Core(
-        os.path.join(
-            os.path.dirname(__file__), "cores", "wb_intercon", "wb_intercon-1.0.core"
-        )
-    )
+    flags = {"tool": "icarus"}
 
     build_root = tempfile.mkdtemp(prefix="export_")
     export_root = os.path.join(build_root, "exported_files")
-    eda_api = Edalizer(
-        core.name,
-        {"tool": "icarus"},
-        [core],
+    work_root = os.path.join(build_root, "work")
+
+    core_dir = os.path.join(os.path.dirname(__file__), "cores")
+
+    cm = CoreManager(Config())
+    cm.add_library(Library("cores", core_dir))
+
+    core = cm.get_core(Vlnv("::wb_intercon"))
+
+    edalizer = Edalizer(
+        toplevel=core.name,
+        flags=flags,
+        core_manager=cm,
         cache_root=None,
-        work_root=os.path.join(build_root, "work"),
+        work_root=work_root,
         export_root=export_root,
-    ).edalize
+        system_name=None,
+    )
+    edalizer.run()
 
     for f in [
         "wb_intercon_1.0/dummy_icarus.v",