From 87437f3e8d8971f826c76d4a5392ae79cc1e5617 Mon Sep 17 00:00:00 2001 From: Dewi Yokelson Date: Sun, 8 Dec 2024 13:15:14 -0800 Subject: [PATCH 1/4] initial checkin, needs merge and initial testing --- modifiers/caliper/modifier.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/modifiers/caliper/modifier.py b/modifiers/caliper/modifier.py index 9eaa29103..f53a28ee6 100644 --- a/modifiers/caliper/modifier.py +++ b/modifiers/caliper/modifier.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 from ramble.modkit import * - +import json def add_mode(mode_name, mode_option, description): mode( @@ -21,7 +21,7 @@ def add_mode(mode_name, mode_option, description): ) -class Caliper(SpackModifier): +class Caliper(SpackModifier, BaseModifier): """Define a modifier for Caliper""" name = "caliper" @@ -30,10 +30,21 @@ class Caliper(SpackModifier): maintainers("pearce8") + # The filename for Caliper output data _cali_datafile = "{experiment_run_dir}/{experiment_name}.cali" + # The filename for metadata forwarded from Benchpark to Caliper + _caliper_metadata_file = "{experiment_run_dir}/{experiment_name}_metadata.json" + + # The metadata dictionary + caliper_metadata = {"application_name": "{application_name}", "experiment_name": "{experiment_name}", + "workload_name": "{workload_name}"} + _default_mode = "time" + # Write out the metadata file once all variables are resolved + register_phase('build_metadata', pipeline='setup', run_after=['make_experiments']) + add_mode( mode_name=_default_mode, mode_option="time.exclusive", @@ -42,7 +53,7 @@ class Caliper(SpackModifier): env_var_modification( "CALI_CONFIG", - "spot(output={}{})".format(_cali_datafile, "${CALI_CONFIG_MODE}"), + "spot(output={}),metadata(file={})".format(_cali_datafile, _caliper_metadata_file), method="set", modes=[_default_mode], ) @@ -83,8 +94,23 @@ class Caliper(SpackModifier): description="Top-down analysis for Intel CPUs (top level)", ) + def _build_metadata(self, workspace, app_inst): + ''' Write the caliper metadata to json ''' + + # Load the Caliper metadata variable from ramble.yaml + experiment_metadata = self.expander.expand_var('caliper_metadata', typed=True, merge_used_stage=False) + self.expander.flush_used_variables() + + # Write + cali_metadata_file = self.expander.expand_var(_caliper_metadata_file) + + with open(cali_metadata_file, "w+") as f: + json.dump(experiment_metadata, stream=f) + + archive_pattern(_cali_datafile) software_spec("caliper", pkg_spec="caliper") required_package("caliper") + \ No newline at end of file From ce9d179a119265cb34d060147793c18fd4da03f6 Mon Sep 17 00:00:00 2001 From: Dewi Yokelson Date: Sun, 8 Dec 2024 19:49:55 -0800 Subject: [PATCH 2/4] almost working, just needs the values from ramble.yaml --- experiments/caliper/experiment.py | 13 +++++++++++++ lib/benchpark/experiment.py | 14 +++++++++++++- modifiers/caliper/modifier.py | 29 ++++++++++++++--------------- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/experiments/caliper/experiment.py b/experiments/caliper/experiment.py index 58423219f..957620078 100644 --- a/experiments/caliper/experiment.py +++ b/experiments/caliper/experiment.py @@ -103,3 +103,16 @@ def get_helper_name_prefix(self): def get_spack_variants(self): return "~caliper" if self.spec.satisfies("caliper=none") else "+caliper" + + def compute_variables_section(self): + """ Add Caliper metadata variables for the ramble.yaml """ + if not self.spec.satisfies("caliper=none"): + return { + "caliper_metadata": { + "application_name": "{application_name}", + "experiment_name": "{experiment_name}", + "workload_name": "{workload_name}" + } + } + else: + return {} \ No newline at end of file diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py index 5b4227e48..fc6ef01d8 100644 --- a/lib/benchpark/experiment.py +++ b/lib/benchpark/experiment.py @@ -47,6 +47,8 @@ def get_helper_name_prefix(self): def get_spack_variants(self): return None + def compute_variables_section(self): + return {} class SingleNode: variant( @@ -240,10 +242,13 @@ def compute_spack_section_wrapper(self): "packages": {k: v for k, v in self.package_specs.items() if v}, "environments": {self.name: {"packages": list(self.package_specs.keys())}}, } + + def compute_variables_section(self): + return {} def compute_ramble_dict(self): # This can be overridden by any subclass that needs more flexibility - return { + ramble_dict = { "ramble": { "include": self.compute_include_section(), "config": self.compute_config_section(), @@ -252,6 +257,13 @@ def compute_ramble_dict(self): "software": self.compute_spack_section_wrapper(), } } + # Add any variables from helper classes + for cls in self.helpers: + additional_vars = cls.compute_variables_section() + if additional_vars: + ramble_dict["ramble"].update({"variables": additional_vars}) + + return ramble_dict def write_ramble_dict(self, filepath): ramble_dict = self.compute_ramble_dict() diff --git a/modifiers/caliper/modifier.py b/modifiers/caliper/modifier.py index f53a28ee6..587798264 100644 --- a/modifiers/caliper/modifier.py +++ b/modifiers/caliper/modifier.py @@ -21,7 +21,7 @@ def add_mode(mode_name, mode_option, description): ) -class Caliper(SpackModifier, BaseModifier): +class Caliper(SpackModifier): """Define a modifier for Caliper""" name = "caliper" @@ -36,10 +36,6 @@ class Caliper(SpackModifier, BaseModifier): # The filename for metadata forwarded from Benchpark to Caliper _caliper_metadata_file = "{experiment_run_dir}/{experiment_name}_metadata.json" - # The metadata dictionary - caliper_metadata = {"application_name": "{application_name}", "experiment_name": "{experiment_name}", - "workload_name": "{workload_name}"} - _default_mode = "time" # Write out the metadata file once all variables are resolved @@ -53,7 +49,7 @@ class Caliper(SpackModifier, BaseModifier): env_var_modification( "CALI_CONFIG", - "spot(output={}),metadata(file={})".format(_cali_datafile, _caliper_metadata_file), + "spot(output={}{}),metadata(file={})".format(_cali_datafile, "${CALI_CONFIG_MODE}", _caliper_metadata_file), method="set", modes=[_default_mode], ) @@ -96,16 +92,19 @@ class Caliper(SpackModifier, BaseModifier): def _build_metadata(self, workspace, app_inst): ''' Write the caliper metadata to json ''' - # Load the Caliper metadata variable from ramble.yaml - experiment_metadata = self.expander.expand_var('caliper_metadata', typed=True, merge_used_stage=False) - self.expander.flush_used_variables() - - # Write - cali_metadata_file = self.expander.expand_var(_caliper_metadata_file) - - with open(cali_metadata_file, "w+") as f: - json.dump(experiment_metadata, stream=f) + # experiment_metadata = self.expander.expand_var('caliper_metadata', typed=True, merge_used_stage=False) + # Error: expand_var() got an unexpected keyword argument 'merge_used_stage' + # TODO: How to get this from the ramble.yaml? + experiment_metadata = self.expander.expand_var('caliper_metadata', typed=True) + #self.expander.flush_used_variables() + # Error: 'Expander' object has no attribute 'flush_used_variables' + + # Write to the Caliper metadata file + cali_metadata_file = self.expander.expand_var(self._caliper_metadata_file) + print(f"writing to %s", cali_metadata_file) + with open(cali_metadata_file, "w") as f: + f.write(json.dumps(experiment_metadata)) archive_pattern(_cali_datafile) From 8031519fbf9d8421d7416782229cb8349323ad03 Mon Sep 17 00:00:00 2001 From: Dewi Yokelson Date: Sun, 8 Dec 2024 20:42:50 -0800 Subject: [PATCH 3/4] adde node.json file on LC to default - can configure this later, shouldn't fail if not found --- modifiers/caliper/modifier.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modifiers/caliper/modifier.py b/modifiers/caliper/modifier.py index 587798264..dafe0ab09 100644 --- a/modifiers/caliper/modifier.py +++ b/modifiers/caliper/modifier.py @@ -49,7 +49,7 @@ class Caliper(SpackModifier): env_var_modification( "CALI_CONFIG", - "spot(output={}{}),metadata(file={})".format(_cali_datafile, "${CALI_CONFIG_MODE}", _caliper_metadata_file), + "spot(output={}{}),metadata(file={}),metadata(file=/etc/node_info.json,keys=\"host.name,host.cluster,host.os\")".format(_cali_datafile, "${CALI_CONFIG_MODE}", _caliper_metadata_file), method="set", modes=[_default_mode], ) @@ -102,7 +102,7 @@ def _build_metadata(self, workspace, app_inst): # Write to the Caliper metadata file cali_metadata_file = self.expander.expand_var(self._caliper_metadata_file) - print(f"writing to %s", cali_metadata_file) + # print(json.dumps(experiment_metadata)) with open(cali_metadata_file, "w") as f: f.write(json.dumps(experiment_metadata)) @@ -111,5 +111,4 @@ def _build_metadata(self, workspace, app_inst): software_spec("caliper", pkg_spec="caliper") - required_package("caliper") - \ No newline at end of file + required_package("caliper") \ No newline at end of file From 75dffd569458aeb4264dae8f479cd5f57c10b916 Mon Sep 17 00:00:00 2001 From: Dewi Yokelson Date: Sun, 22 Dec 2024 17:21:46 -0800 Subject: [PATCH 4/4] pipeline works, now to just add as much data as possible --- checkout-versions.yaml | 2 +- experiments/caliper/experiment.py | 9 ++++----- lib/benchpark/experiment.py | 7 ++++--- modifiers/caliper/modifier.py | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/checkout-versions.yaml b/checkout-versions.yaml index 0fdff240f..2cc45000b 100644 --- a/checkout-versions.yaml +++ b/checkout-versions.yaml @@ -4,5 +4,5 @@ # SPDX-License-Identifier: Apache-2.0 versions: - ramble: a838e7f # develop on 7/17/2024 (newer than 0.5.0 release) + ramble: 39345d8 # develop on 12/17/2024 (newer than 0.5.0 release) spack: 0b3b49b # develop-2024-06-28 (newer than 0.22.0 release) diff --git a/experiments/caliper/experiment.py b/experiments/caliper/experiment.py index 957620078..f78ed7654 100644 --- a/experiments/caliper/experiment.py +++ b/experiments/caliper/experiment.py @@ -107,12 +107,11 @@ def get_spack_variants(self): def compute_variables_section(self): """ Add Caliper metadata variables for the ramble.yaml """ if not self.spec.satisfies("caliper=none"): - return { - "caliper_metadata": { - "application_name": "{application_name}", + metadata_dict = {"application_name": "{application_name}", "experiment_name": "{experiment_name}", - "workload_name": "{workload_name}" - } + "workload_name": "{workload_name}"} + return { + "caliper_metadata": metadata_dict } else: return {} \ No newline at end of file diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py index fc6ef01d8..8f6208c50 100644 --- a/lib/benchpark/experiment.py +++ b/lib/benchpark/experiment.py @@ -258,10 +258,11 @@ def compute_ramble_dict(self): } } # Add any variables from helper classes + self.additional_vars = {} for cls in self.helpers: - additional_vars = cls.compute_variables_section() - if additional_vars: - ramble_dict["ramble"].update({"variables": additional_vars}) + self.additional_vars = cls.compute_variables_section() + if self.additional_vars: + ramble_dict["ramble"].update({"variables": self.additional_vars}) return ramble_dict diff --git a/modifiers/caliper/modifier.py b/modifiers/caliper/modifier.py index dafe0ab09..1f435cfb2 100644 --- a/modifiers/caliper/modifier.py +++ b/modifiers/caliper/modifier.py @@ -21,7 +21,7 @@ def add_mode(mode_name, mode_option, description): ) -class Caliper(SpackModifier): +class Caliper(BasicModifier): """Define a modifier for Caliper""" name = "caliper" @@ -93,18 +93,18 @@ class Caliper(SpackModifier): def _build_metadata(self, workspace, app_inst): ''' Write the caliper metadata to json ''' # Load the Caliper metadata variable from ramble.yaml - # experiment_metadata = self.expander.expand_var('caliper_metadata', typed=True, merge_used_stage=False) - # Error: expand_var() got an unexpected keyword argument 'merge_used_stage' - # TODO: How to get this from the ramble.yaml? - experiment_metadata = self.expander.expand_var('caliper_metadata', typed=True) - #self.expander.flush_used_variables() - # Error: 'Expander' object has no attribute 'flush_used_variables' + experiment_metadata = app_inst.expander.expand_var_name('caliper_metadata', typed=True, merge_used_stage=False) + app_inst.expander.flush_used_variable_stage() + + # rebuild dictionary with expanded variables + cali_metadata = {} + for key, val in experiment_metadata.items(): + cali_metadata[key] = app_inst.expander.expand_var(val) # Write to the Caliper metadata file cali_metadata_file = self.expander.expand_var(self._caliper_metadata_file) - # print(json.dumps(experiment_metadata)) with open(cali_metadata_file, "w") as f: - f.write(json.dumps(experiment_metadata)) + f.write(json.dumps(cali_metadata)) archive_pattern(_cali_datafile)