From ed09c2e72ed0e5df110a3be82f97574cdca5f117 Mon Sep 17 00:00:00 2001 From: Philipp Wagner Date: Mon, 6 Apr 2020 16:25:09 +0100 Subject: [PATCH] Refactor Edalizer class Refactor this class to split out functionality into smaller sub-functions to make it easier to extend. In the process, remove some Python 2 functionality we don't need any more. The call to the depenceny resolver moved into the Edalize class as well to provide a slightly cleaner interface: a CoreManager instance, and the top-level core. Updated the tests to fulfill the new requirement to pass a CoreManager instance to Edalizer. Move some test files into separate directories to keep core libraries separate and avoid spilling between tests. Thanks to Rupert for working on the tests. Co-authored-by: Rupert Swarbrick --- fusesoc/edalizer.py | 152 +++++++++++++----- fusesoc/main.py | 17 +- .../misc/{ => generate}/generate.core | 0 .../misc/{ => generate}/generators.core | 0 .../misc/{ => generate}/testgen.py | 0 .../misc/{ => copytocore}/copytocore.core | 0 tests/cores/misc/copytocore/dummy.tcl | 0 .../cores/misc/copytocore/subdir/dummy.extra | 0 tests/test_capi2.py | 12 +- tests/test_coremanager.py | 63 ++++++-- tests/test_edalizer.py | 29 ++-- 11 files changed, 193 insertions(+), 80 deletions(-) rename tests/capi2_cores/misc/{ => generate}/generate.core (100%) rename tests/capi2_cores/misc/{ => generate}/generators.core (100%) rename tests/capi2_cores/misc/{ => generate}/testgen.py (100%) rename tests/cores/misc/{ => copytocore}/copytocore.core (100%) create mode 100644 tests/cores/misc/copytocore/dummy.tcl create mode 100644 tests/cores/misc/copytocore/subdir/dummy.extra diff --git a/fusesoc/edalizer.py b/fusesoc/edalizer.py index 897ffde2..8faab886 100644 --- a/fusesoc/edalizer.py +++ b/fusesoc/edalizer.py @@ -6,6 +6,7 @@ from fusesoc import utils from fusesoc.vlnv import Vlnv from fusesoc.utils import merge_dict +from fusesoc.coremanager import DependencyError logger = logging.getLogger(__name__) @@ -21,59 +22,148 @@ def __call__(self, parser, namespace, values, option_string=None): class Edalizer: def __init__( self, - vlnv, + toplevel, flags, - cores, cache_root, work_root, + core_manager, export_root=None, system_name=None, ): - if os.path.exists(work_root): - for f in os.listdir(work_root): - if os.path.isdir(os.path.join(work_root, f)): - shutil.rmtree(os.path.join(work_root, f)) + logger.debug("Building EDA API") + + self.toplevel = toplevel + self.flags = flags + self.cache_root = cache_root + self.core_manager = core_manager + self.work_root = work_root + self.export_root = export_root + self.system_name = system_name + + self._prepare_work_root() + + self.generators = {} + + self._generated_cores = [] + + @property + def cores(self): + return self.resolved_cores + self._generated_cores + + @property + def resolved_cores(self): + """ Get a list of all "used" cores after the dependency resolution """ + try: + return self.core_manager.get_depends(self.toplevel, self.flags) + except DependencyError as e: + logger.error( + e.msg + "\nFailed to resolve dependencies for {}".format(self.toplevel) + ) + exit(1) + except SyntaxError as e: + logger.error(e.msg) + exit(1) + + @property + def discovered_cores(self): + """ Get a list of all cores found by fusesoc """ + return self.core_manager.db.find() + + def run(self): + """ Run all steps to create a EDA API YAML file """ + + # Run the setup task on all cores (fetch and patch them as needed) + self.setup_cores() + + # Get all generators defined in any of the cores + self.extract_generators() + + # Run all generators. Generators can create new cores, which are added + # to the list of available cores. + self.run_generators() + + # Create EDA API file contents + self.create_eda_api_struct() + + def _core_flags(self, core): + """ Get flags for a specific core """ + + core_flags = self.flags.copy() + core_flags["is_toplevel"] = core.name == self.toplevel + return core_flags + + def _prepare_work_root(self): + if os.path.exists(self.work_root): + for f in os.listdir(self.work_root): + if os.path.isdir(os.path.join(self.work_root, f)): + shutil.rmtree(os.path.join(self.work_root, f)) else: - os.remove(os.path.join(work_root, f)) + os.remove(os.path.join(self.work_root, f)) else: - os.makedirs(work_root) + os.makedirs(self.work_root) - logger.debug("Building EDA API") + def setup_cores(self): + """ Setup cores: fetch resources, patch them, etc. """ + for core in self.cores: + logger.info("Preparing " + str(core.name)) + core.setup() + def extract_generators(self): + """ Get all registered generators from the cores """ generators = {} + for core in self.cores: + logger.debug("Searching for generators in " + str(core.name)) + core_flags = self._core_flags(core) + + if hasattr(core, "get_generators"): + core_generators = core.get_generators(core_flags) + logger.debug("Found generators: %s" % (core_generators,)) + generators.update(core_generators) + self.generators = generators + + def run_generators(self): + """ Run all generators """ + for core in self.cores: + logger.debug("Running generators in " + str(core.name)) + core_flags = self._core_flags(core) + + if hasattr(core, "get_ttptttg"): + for ttptttg_data in core.get_ttptttg(core_flags): + _ttptttg = Ttptttg(ttptttg_data, core, self.generators,) + for gen_core in _ttptttg.generate(self.cache_root): + gen_core.pos = _ttptttg.pos + self._generated_cores.append(gen_core) + + def create_eda_api_struct(self): first_snippets = [] snippets = [] last_snippets = [] - _flags = flags.copy() - core_queue = cores[:] - core_queue.reverse() - while core_queue: + for core in self.cores: snippet = {} - core = core_queue.pop() - logger.info("Preparing " + str(core.name)) - core.setup() logger.debug("Collecting EDA API parameters from {}".format(str(core.name))) - _flags["is_toplevel"] = core.name == vlnv + _flags = self._core_flags(core) # Extract direct dependencies snippet["dependencies"] = {str(core.name): core.direct_deps} # Extract files - if export_root: - files_root = os.path.join(export_root, core.sanitized_name) + if self.export_root: + files_root = os.path.join(self.export_root, core.sanitized_name) core.export(files_root, _flags) else: files_root = core.files_root - rel_root = os.path.relpath(files_root, work_root) + rel_root = os.path.relpath(files_root, self.work_root) # Extract parameters snippet["parameters"] = core.get_parameters(_flags) # Extract tool options - snippet["tool_options"] = {flags["tool"]: core.get_tool_options(_flags)} + snippet["tool_options"] = { + self.flags["tool"]: core.get_tool_options(_flags) + } # Extract scripts snippet["hooks"] = core.get_scripts(rel_root, _flags) @@ -82,7 +172,7 @@ def __init__( for file in core.get_files(_flags): if file.copyto: _name = file.copyto - dst = os.path.join(work_root, _name) + dst = os.path.join(self.work_root, _name) _dstdir = os.path.dirname(dst) if not os.path.exists(_dstdir): os.makedirs(_dstdir) @@ -117,18 +207,6 @@ def __init__( } ) - # Extract generators if defined in CAPI - if hasattr(core, "get_generators"): - generators.update(core.get_generators(_flags)) - - # Run generators - if hasattr(core, "get_ttptttg"): - for ttptttg_data in core.get_ttptttg(_flags): - _ttptttg = Ttptttg(ttptttg_data, core, generators) - for gen_core in _ttptttg.generate(cache_root): - gen_core.pos = _ttptttg.pos - core_queue.append(gen_core) - if hasattr(core, "pos"): if core.pos == "first": first_snippets.append(snippet) @@ -139,16 +217,16 @@ def __init__( else: snippets.append(snippet) - top_core = cores[-1] + top_core = self.resolved_cores[-1] self.edalize = { "version": "0.2.1", "dependencies": {}, "files": [], "hooks": {}, - "name": system_name or top_core.sanitized_name, + "name": self.system_name or top_core.sanitized_name, "parameters": {}, "tool_options": {}, - "toplevel": top_core.get_toplevel(flags), + "toplevel": top_core.get_toplevel(self.flags), "vpi": [], } diff --git a/fusesoc/main.py b/fusesoc/main.py index bf6dd209..aa7ce641 100644 --- a/fusesoc/main.py +++ b/fusesoc/main.py @@ -409,26 +409,17 @@ def run_backend( do_configure = True if do_configure: - try: - cores = cm.get_depends(core.name, flags) - except DependencyError as e: - logger.error( - e.msg + "\nFailed to resolve dependencies for {}".format(system) - ) - exit(1) - except SyntaxError as e: - logger.error(e.msg) - exit(1) try: edalizer = Edalizer( - core.name, - flags, - cores, + toplevel=core.name, + flags=flags, + core_manager=cm, cache_root=cm.config.cache_root, work_root=work_root, export_root=export_root, system_name=system_name, ) + edalizer.run() backend_class = get_edatool(tool) edalizer.parse_args(backend_class, backendargs) diff --git a/tests/capi2_cores/misc/generate.core b/tests/capi2_cores/misc/generate/generate.core similarity index 100% rename from tests/capi2_cores/misc/generate.core rename to tests/capi2_cores/misc/generate/generate.core diff --git a/tests/capi2_cores/misc/generators.core b/tests/capi2_cores/misc/generate/generators.core similarity index 100% rename from tests/capi2_cores/misc/generators.core rename to tests/capi2_cores/misc/generate/generators.core diff --git a/tests/capi2_cores/misc/testgen.py b/tests/capi2_cores/misc/generate/testgen.py similarity index 100% rename from tests/capi2_cores/misc/testgen.py rename to tests/capi2_cores/misc/generate/testgen.py diff --git a/tests/cores/misc/copytocore.core b/tests/cores/misc/copytocore/copytocore.core similarity index 100% rename from tests/cores/misc/copytocore.core rename to tests/cores/misc/copytocore/copytocore.core diff --git a/tests/cores/misc/copytocore/dummy.tcl b/tests/cores/misc/copytocore/dummy.tcl new file mode 100644 index 00000000..e69de29b diff --git a/tests/cores/misc/copytocore/subdir/dummy.extra b/tests/cores/misc/copytocore/subdir/dummy.extra new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_capi2.py b/tests/test_capi2.py index ce57ceae..ee54a100 100644 --- a/tests/test_capi2.py +++ b/tests/test_capi2.py @@ -186,7 +186,7 @@ def test_capi2_get_files(): def test_capi2_get_generators(): from fusesoc.core import Core - core = Core(os.path.join(cores_dir, "generators.core")) + core = Core(os.path.join(cores_dir, "generate", "generators.core")) generators = core.get_generators({}) assert len(generators) == 1 @@ -441,7 +441,7 @@ def test_capi2_get_toplevel(): def test_capi2_get_ttptttg(): from fusesoc.core import Core - core = Core(os.path.join(cores_dir, "generate.core")) + core = Core(os.path.join(cores_dir, "generate", "generate.core")) flags = {"is_toplevel": True} expected = [ @@ -517,8 +517,12 @@ def test_capi2_get_work_root(): def test_capi2_info(): from fusesoc.core import Core - for core_name in ["targets", "generators"]: - core_file = os.path.join(tests_dir, "capi2_cores", "misc", core_name + ".core") + dirs_and_corenames = [("", "targets"), ("generate", "generators")] + + for subdir, core_name in dirs_and_corenames: + core_file = os.path.join( + tests_dir, "capi2_cores", "misc", subdir, core_name + ".core" + ) core = Core(core_file) gen_info = "\n".join( diff --git a/tests/test_coremanager.py b/tests/test_coremanager.py index 157559d0..6791e2b1 100644 --- a/tests/test_coremanager.py +++ b/tests/test_coremanager.py @@ -52,17 +52,36 @@ def test_copyto(): import os import tempfile + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager from fusesoc.edalizer import Edalizer - from fusesoc.core import Core + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv - core = Core( - os.path.join(os.path.dirname(__file__), "cores", "misc", "copytocore.core") - ) flags = {"tool": "icarus"} work_root = tempfile.mkdtemp(prefix="copyto_") - eda_api = Edalizer(core.name, flags, [core], None, work_root, None).edalize + core_dir = os.path.join(os.path.dirname(__file__), "cores", "misc", "copytocore") + lib = Library("misc", core_dir) + + cm = CoreManager(Config()) + cm.add_library(lib) + + core = cm.get_core(Vlnv("::copytocore")) + + edalizer = Edalizer( + toplevel=core.name, + flags=flags, + core_manager=cm, + cache_root=None, + work_root=work_root, + export_root=None, + system_name=None, + ) + edalizer.run() + + eda_api = edalizer.edalize assert eda_api["files"] == [ { @@ -88,25 +107,35 @@ def test_export(): import os import tempfile + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager from fusesoc.edalizer import Edalizer - from fusesoc.core import Core + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv - core = Core( - os.path.join( - os.path.dirname(__file__), "cores", "wb_intercon", "wb_intercon-1.0.core" - ) - ) + flags = {"tool": "icarus"} build_root = tempfile.mkdtemp(prefix="export_") export_root = os.path.join(build_root, "exported_files") - eda_api = Edalizer( - core.name, - {"tool": "icarus"}, - [core], + work_root = os.path.join(build_root, "work") + + core_dir = os.path.join(os.path.dirname(__file__), "cores") + + cm = CoreManager(Config()) + cm.add_library(Library("cores", core_dir)) + + core = cm.get_core(Vlnv("::wb_intercon")) + + edalizer = Edalizer( + toplevel=core.name, + flags=flags, + core_manager=cm, cache_root=None, - work_root=os.path.join(build_root, "work"), + work_root=work_root, export_root=export_root, - ).edalize + system_name=None, + ) + edalizer.run() for f in [ "wb_intercon_1.0/dummy_icarus.v", diff --git a/tests/test_edalizer.py b/tests/test_edalizer.py index 3e3d5bc0..9ad24426 100644 --- a/tests/test_edalizer.py +++ b/tests/test_edalizer.py @@ -2,26 +2,37 @@ def test_generators(): import os import tempfile + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager from fusesoc.edalizer import Edalizer - from fusesoc.core import Core + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv tests_dir = os.path.dirname(__file__) - cores_dir = os.path.join(tests_dir, "capi2_cores", "misc") + cores_dir = os.path.join(tests_dir, "capi2_cores", "misc", "generate") - core1 = Core(os.path.join(cores_dir, "generators.core")) - core2 = Core(os.path.join(cores_dir, "generate.core")) + lib = Library("edalizer", cores_dir) + + cm = CoreManager(Config()) + cm.add_library(lib) + + core = cm.get_core(Vlnv("::generate")) build_root = tempfile.mkdtemp(prefix="export_") cache_root = tempfile.mkdtemp(prefix="export_cache_") export_root = os.path.join(build_root, "exported_files") - eda_api = Edalizer( - core2.name, - {"tool": "icarus"}, - [core1, core2], + + edalizer = Edalizer( + toplevel=core.name, + flags={"tool": "icarus"}, + core_manager=cm, cache_root=cache_root, work_root=os.path.join(build_root, "work"), export_root=export_root, - ).edalize + system_name=None, + ) + edalizer.run() + gendir = os.path.join( cache_root, "generated", "generate-testgenerate_without_params_0" )