From be67182a3475eec3b0a7a003eb721c19540476ca Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Wed, 6 Mar 2024 22:18:37 -0600 Subject: [PATCH 1/9] improve or_matched; add slc test --- pyincore/dfr3service.py | 14 +++----- .../buildingdamage/test_slc_buildingdamage.py | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 tests/pyincore/analyses/buildingdamage/test_slc_buildingdamage.py diff --git a/pyincore/dfr3service.py b/pyincore/dfr3service.py index d8b89c807..b87485a70 100644 --- a/pyincore/dfr3service.py +++ b/pyincore/dfr3service.py @@ -352,16 +352,10 @@ def _property_match_legacy(rules, properties): else: # rules = [[A and B], OR [C and D], OR [E and F]] - or_matched = [False for i in range(len(rules))] # initiate all false state outer list - for i, and_rules in enumerate(rules): - and_matched = [False for j in range(len(and_rules))] # initialte all false state for inner list - for j, rule in enumerate(and_rules): - # evaluate, return True or False. And place it in the corresponding place - and_matched[j] = Dfr3Service._eval_criterion(rule, properties) - - # for inner list, AND boolean applied - if all(and_matched): - or_matched[i] = True + or_matched = [ + all(map(lambda rule: Dfr3Service._eval_criterion(rule, properties), and_rules)) + for and_rules in rules + ] # for outer list, OR boolean is appied return any(or_matched) diff --git a/tests/pyincore/analyses/buildingdamage/test_slc_buildingdamage.py b/tests/pyincore/analyses/buildingdamage/test_slc_buildingdamage.py new file mode 100644 index 000000000..ad7215cd4 --- /dev/null +++ b/tests/pyincore/analyses/buildingdamage/test_slc_buildingdamage.py @@ -0,0 +1,36 @@ +from pyincore import IncoreClient, FragilityService, MappingSet, Earthquake, HazardService, DataService +from pyincore.analyses.buildingdamage import BuildingDamage +import time + + +if __name__ == "__main__": + client = IncoreClient() + + # Initiate fragility service + fragility_services = FragilityService(client) + hazard_services = HazardService(client) + data_services = DataService(client) + + # Analysis setup + start_time = time.time() + bldg_dmg = BuildingDamage(client) + + mapping_set = MappingSet(fragility_services.get_mapping("6309005ad76c6d0e1f6be081")) + bldg_dmg.set_input_dataset('dfr3_mapping_set', mapping_set) + + bldg_dmg.load_remote_input_dataset("buildings", "62fea288f5438e1f8c515ef8") # Salt Lake County All Building + bldg_dmg.set_parameter("result_name", "SLC_bldg_dmg_no_retrofit-withLIQ7.1") + + eq = Earthquake.from_hazard_service("640a03ea73a1642180262450", hazard_services) # Mw 7.1 + # eq = Earthquake.from_hazard_service("64108b6486a52d419dd69a41", hazard_services) # Mw 7.0 + bldg_dmg.set_input_hazard("hazard", eq) + + bldg_dmg.set_parameter("use_liquefaction", True) + bldg_dmg.set_parameter("liquefaction_geology_dataset_id", "62fe9ab685ac6b569e372429") + bldg_dmg.set_parameter("num_cpu", 8) + + # Run building damage without liquefaction + bldg_dmg.run_analysis() + + end_time = time.time() + print(f"total runtime: {end_time - start_time}") \ No newline at end of file From add443f34aef9228993f082c5a2837f45ce32f35 Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Wed, 6 Mar 2024 22:25:15 -0600 Subject: [PATCH 2/9] improve payload loop --- pyincore/analyses/buildingdamage/buildingdamage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyincore/analyses/buildingdamage/buildingdamage.py b/pyincore/analyses/buildingdamage/buildingdamage.py index b6a8b863c..36dff2268 100755 --- a/pyincore/analyses/buildingdamage/buildingdamage.py +++ b/pyincore/analyses/buildingdamage/buildingdamage.py @@ -202,9 +202,10 @@ def building_damage_analysis_bulk_input(self, buildings, hazards, hazard_types, # worst code I have ever written # e.g. 1.04 Sec Sa --> 1.04 SA --> 1.0 SA for payload, response in zip(values_payload, hazard_vals): - for i in range(len(payload["demands"])): - adjust_demand_types_mapping[response["demands"][i]] = adjust_demand_types_mapping[payload[ - "demands"][i]] + adjust_demand_types_mapping.update({ + response_demand: adjust_demand_types_mapping[payload_demand] + for payload_demand, response_demand in zip(payload["demands"], response["demands"]) + }) # record hazard value for each hazard type for later calcu multihazard_vals[hazard_type] = hazard_vals From 6fe429c5e41f534623668020d81d215f3fbcbb17 Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Wed, 6 Mar 2024 22:55:23 -0600 Subject: [PATCH 3/9] remove keys() as not needed --- pyincore/dfr3service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyincore/dfr3service.py b/pyincore/dfr3service.py index b87485a70..0dceea722 100644 --- a/pyincore/dfr3service.py +++ b/pyincore/dfr3service.py @@ -219,7 +219,7 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: Opt # [[ and ] or [ and ]] if isinstance(m.rules, list): if self._property_match_legacy(rules=m.rules, properties=inventory["properties"]): - if retrofit_entry_key is not None and retrofit_entry_key in m.entry.keys(): + if retrofit_entry_key is not None and retrofit_entry_key in m.entry: curve = m.entry[retrofit_entry_key] else: curve = m.entry[entry_key] @@ -236,7 +236,7 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: Opt # {"AND": [xx, "OR": [yy, yy], "AND": {"OR":["zz", "zz"]]} elif isinstance(m.rules, dict): if self._property_match(rules=m.rules, properties=inventory["properties"]): - if retrofit_entry_key is not None and retrofit_entry_key in m.entry.keys(): + if retrofit_entry_key is not None and retrofit_entry_key in m.entry: curve = m.entry[retrofit_entry_key] else: curve = m.entry[entry_key] From 895aa131e79e12f6c531100ddb9fa334f942d212 Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Wed, 6 Mar 2024 23:25:10 -0600 Subject: [PATCH 4/9] remove keys to speed up --- pyincore/dfr3service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyincore/dfr3service.py b/pyincore/dfr3service.py index 0dceea722..fa07a9de6 100644 --- a/pyincore/dfr3service.py +++ b/pyincore/dfr3service.py @@ -419,18 +419,18 @@ def _eval_criterion(rule, properties): # e.g. "java.lang.String struct_typ EQUALS W2" rule_type = elements[0] # e.g. int, str, double, java.lang.String, etc... - if rule_type not in known_types.keys(): + if rule_type not in known_types: raise ValueError(rule_type + " Unknown. Cannot parse the rules of this mapping!") rule_key = elements[1] # e.g. no_storeis, year_built, etc... rule_operator = elements[2] # e.g. EQ, GE, LE, EQUALS - if rule_operator not in known_operators.keys(): + if rule_operator not in known_operators: raise ValueError(rule_operator + " Unknown. Cannot parse the rules of this mapping!") rule_value = elements[3].strip('\'').strip('\"') - if rule_key in properties.keys(): + if rule_key in properties: # validate if the rule is written correctly by comparing variable type, e.g. no_stories properties # should be integer if isinstance(properties[rule_key], eval(known_types[rule_type])): From 1ff479d9fc7c8405e435b8d449b1adacbfa7eed0 Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Thu, 7 Mar 2024 13:54:29 -0600 Subject: [PATCH 5/9] speed up using pre-filter on mapped/unmapped buildings --- .../analyses/buildingdamage/buildingdamage.py | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/pyincore/analyses/buildingdamage/buildingdamage.py b/pyincore/analyses/buildingdamage/buildingdamage.py index 36dff2268..e776e71c7 100755 --- a/pyincore/analyses/buildingdamage/buildingdamage.py +++ b/pyincore/analyses/buildingdamage/buildingdamage.py @@ -165,36 +165,35 @@ def building_damage_analysis_bulk_input(self, buildings, hazards, hazard_types, values_payload = [] values_payload_liq = [] # for liquefaction, if used - unmapped_buildings = [] - mapped_buildings = [] - for b in buildings: + + # Pre-filter buildings that are in fragility_sets to reduce the number of iterations + mapped_buildings = [b for b in buildings if b["id"] in fragility_sets] + unmapped_buildings = [b for b in buildings if b["id"] not in fragility_sets] + + for b in mapped_buildings: bldg_id = b["id"] - if bldg_id in fragility_sets: - location = GeoUtil.get_location(b) - loc = str(location.y) + "," + str(location.x) - demands, units, adjusted_to_original = \ - AnalysisUtil.get_hazard_demand_types_units(b, - fragility_sets[bldg_id], - hazard_type, - allowed_demand_types) - adjust_demand_types_mapping.update(adjusted_to_original) - value = { - "demands": demands, - "units": units, + location = GeoUtil.get_location(b) + loc = str(location.y) + "," + str(location.x) + demands, units, adjusted_to_original = \ + AnalysisUtil.get_hazard_demand_types_units(b, + fragility_sets[bldg_id], + hazard_type, + allowed_demand_types) + adjust_demand_types_mapping.update(adjusted_to_original) + value = { + "demands": demands, + "units": units, + "loc": loc + } + values_payload.append(value) + + if use_liquefaction and geology_dataset_id is not None: + value_liq = { + "demands": [""], + "units": [""], "loc": loc } - values_payload.append(value) - mapped_buildings.append(b) - - if use_liquefaction and geology_dataset_id is not None: - value_liq = { - "demands": [""], - "units": [""], - "loc": loc - } - values_payload_liq.append(value_liq) - else: - unmapped_buildings.append(b) + values_payload_liq.append(value_liq) hazard_vals = hazard.read_hazard_values(values_payload, self.hazardsvc) From 5410e1c57b4a577ff57f42dd30b57052b26bdf0f Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Thu, 7 Mar 2024 14:11:22 -0600 Subject: [PATCH 6/9] improve lens --- pyincore/utils/analysisutil.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyincore/utils/analysisutil.py b/pyincore/utils/analysisutil.py index 669bd4899..fabf97ce9 100644 --- a/pyincore/utils/analysisutil.py +++ b/pyincore/utils/analysisutil.py @@ -261,14 +261,17 @@ def adjust_damage_for_liquefaction(limit_state_probabilities, ground_failure_pro keys = list(limit_state_probabilities.keys()) adjusted_limit_state_probabilities = collections.OrderedDict() - for i in range(len(keys)): + ground_failure_probabilities_len = len(ground_failure_probabilities) + keys_len = len(keys) + + for i in range(keys_len): # check and see...if we are trying to use the last ground failure # number for something other than the # last limit-state-probability, then we should use the # second-to-last probability of ground failure instead. - if i > len(ground_failure_probabilities) - 1: - prob_ground_failure = ground_failure_probabilities[len(ground_failure_probabilities) - 2] + if i > ground_failure_probabilities_len - 1: + prob_ground_failure = ground_failure_probabilities[ground_failure_probabilities_len - 2] else: prob_ground_failure = ground_failure_probabilities[i] From 7aadf6afda4e4703dbe0fb744a546dc482315b58 Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Thu, 7 Mar 2024 14:21:56 -0600 Subject: [PATCH 7/9] =?UTF-8?q?Added=20cache=20to=20store=20fragility=20ma?= =?UTF-8?q?tches=20and=20the=20inventory=20attributes=20s=E2=80=A6=20(#517?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added cache to store fragility matches and the inventory attributes so like structures can pull from the cache * Add caching for both new and old rule sets * Updated to handle both old and new rule parsing when building the cache --- pyincore/dfr3service.py | 162 ++++++++++++++++++++++++++++++++-------- 1 file changed, 129 insertions(+), 33 deletions(-) diff --git a/pyincore/dfr3service.py b/pyincore/dfr3service.py index fa07a9de6..4e37e4b08 100644 --- a/pyincore/dfr3service.py +++ b/pyincore/dfr3service.py @@ -190,6 +190,7 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: Opt """ dfr3_sets = {} + dfr3_sets_cache = {} # find default mapping entry key if not provided if entry_key is None: @@ -213,41 +214,56 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: Opt # if retrofit key exist, use retrofit key otherwise use default key retrofit_entry_key = inventory["properties"]["retrofit_k"] if "retrofit_k" in \ inventory["properties"] else None + + cached_curve = self._check_cache(dfr3_sets_cache, inventory["properties"]) - for m in mapping.mappings: - # for old format rule matching [[]] - # [[ and ] or [ and ]] - if isinstance(m.rules, list): - if self._property_match_legacy(rules=m.rules, properties=inventory["properties"]): - if retrofit_entry_key is not None and retrofit_entry_key in m.entry: - curve = m.entry[retrofit_entry_key] - else: - curve = m.entry[entry_key] - dfr3_sets[inventory['id']] = curve + if cached_curve is not None: + dfr3_sets[inventory['id']] = cached_curve - # if it's string:id; then need to fetch it from remote and cast to dfr3curve object - if isinstance(curve, str) and curve not in matched_curve_ids: - matched_curve_ids.append(curve) - - # use the first match - break - - # for new format rule matching {"AND/OR":[]} - # {"AND": [xx, "OR": [yy, yy], "AND": {"OR":["zz", "zz"]]} - elif isinstance(m.rules, dict): - if self._property_match(rules=m.rules, properties=inventory["properties"]): - if retrofit_entry_key is not None and retrofit_entry_key in m.entry: - curve = m.entry[retrofit_entry_key] - else: - curve = m.entry[entry_key] - dfr3_sets[inventory['id']] = curve - - # if it's string:id; then need to fetch it from remote and cast to dfr3 curve object - if isinstance(curve, str) and curve not in matched_curve_ids: - matched_curve_ids.append(curve) - - # use the first match - break + else: + for m in mapping.mappings: + # for old format rule matching [[]] + # [[ and ] or [ and ]] + if isinstance(m.rules, list): + if self._property_match_legacy(rules=m.rules, properties=inventory["properties"]): + if retrofit_entry_key is not None and retrofit_entry_key in m.entry: + curve = m.entry[retrofit_entry_key] + else: + curve = m.entry[entry_key] + + dfr3_sets[inventory['id']] = curve + + matched_properties_dict = self._convert_properties_to_dict(m.rules, inventory["properties"]) + # Add the matched inventory properties so other matching inventory can avoid rule matching + dfr3_sets_cache[curve] = matched_properties_dict + + # if it's string:id; then need to fetch it from remote and cast to dfr3curve object + if isinstance(curve, str) and curve not in matched_curve_ids: + matched_curve_ids.append(curve) + + # use the first match + break + + # for new format rule matching {"AND/OR":[]} + # {"AND": [xx, "OR": [yy, yy], "AND": {"OR":["zz", "zz"]]} + elif isinstance(m.rules, dict): + if self._property_match(rules=m.rules, properties=inventory["properties"]): + if retrofit_entry_key is not None and retrofit_entry_key in m.entry: + curve = m.entry[retrofit_entry_key] + else: + curve = m.entry[entry_key] + dfr3_sets[inventory['id']] = curve + + matched_properties_dict = self._convert_properties_to_dict(m.rules, inventory["properties"]) + # Add the matched inventory properties so other matching inventory can avoid rule matching + dfr3_sets_cache[curve] = matched_properties_dict + + # if it's string:id; then need to fetch it from remote and cast to dfr3 curve object + if isinstance(curve, str) and curve not in matched_curve_ids: + matched_curve_ids.append(curve) + + # use the first match + break batch_dfr3_sets = self.batch_get_dfr3_set(matched_curve_ids) @@ -333,6 +349,67 @@ def match_list_of_dicts(self, mapping: MappingSet, inventories: list, entry_key: return dfr3_sets + @staticmethod + def _check_cache(dfr3_sets_dict, properties): + """A method to see if we already have matched an inventory with the same properties to a fragility curve + + Args: + dfr3_sets_dict (dict): {"fragility-curve-id-1": {"struct_typ": "W1", "no_stories": "2"}, etc.} + properties (obj): A fiona Properties object that contains properties of the inventory row. + + Returns: + Fragility curve id if a match is found + + """ + if not dfr3_sets_dict: + return None + + for entry_key in dfr3_sets_dict: + inventory_dict = {} + entry_dict = dfr3_sets_dict[entry_key] + for rule_key in entry_dict: + inventory_dict[rule_key] = properties[rule_key] + + if entry_dict == inventory_dict: + return entry_key + + @staticmethod + def _convert_properties_to_dict(rules, properties): + """A method to convert properties to a dictionary with only the matched values in the rule set + + Args: + rules (obj): [[A and B] or [C and D]] + properties (dict): A dictionary that contains properties of the inventory row. + + Returns: + Dictionary of property values for the inventory object so the matched fragility can be cached + + """ + matched_properties = {} + # Handle legacy rules + if isinstance(rules, list): + for i, and_rules in enumerate(rules): + for j, rule in enumerate(and_rules): + matched_properties.update(Dfr3Service._eval_property_from_inventory(rule, properties)) + elif isinstance(rules, dict): + # Handles new style of rules + boolean = list(rules.keys())[0] # AND or OR + criteria = rules[boolean] + + for criterion in criteria: + # Recursively parse and evaluate the rules with boolean + if isinstance(criterion, dict): + for criteria in criterion: + for rule_criteria in criterion[criteria]: + matched_properties.update(Dfr3Service._eval_property_from_inventory(rule_criteria, + properties)) + elif isinstance(criterion, str): + matched_properties.update(Dfr3Service._eval_property_from_inventory(criterion, properties)) + else: + raise ValueError("Cannot evaluate criterion, unsupported format!") + + return matched_properties + @staticmethod def _property_match_legacy(rules, properties): """A method to determine whether current set of rules rules applied to the inventory row (legacy rule format). @@ -398,6 +475,25 @@ def _property_match(rules, properties): else: raise ValueError("boolean " + boolean + " not supported!") + @staticmethod + def _eval_property_from_inventory(rule, properties): + """A method to evaluate individual rule and get the property from the inventory properties. + + Args: + rule (str): # e.g. "int no_stories EQ 1", + properties (dict): dictionary of properties of an inventory item. e.g. {"guid":xxx, + "num_stories":xxx, ...} + + Returns: + dictionary entry with the inventory property value that matched the rule + + """ + elements = rule.split(" ", 3) + property_key = elements[1] + + matched_props = {property_key: properties[property_key]} + return matched_props + @staticmethod def _eval_criterion(rule, properties): """A method to evaluate individual rule and see if it appies to a certain inventory row. From bfbca799c28d2fbfbb5ea46a92ba846fe0c49b66 Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang Date: Thu, 7 Mar 2024 14:30:41 -0600 Subject: [PATCH 8/9] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8772b487..f797bcbeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Retrofitted Building Damage [#469](https://github.com/IN-CORE/pyincore/issues/469) +- Optimize building damage performance [#513](https://github.com/IN-CORE/pyincore/issues/513) ## [1.16.0] - 2024-02-07 From 78eb5945cbe076566583586c934a66c35cf5831c Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Thu, 7 Mar 2024 15:59:09 -0600 Subject: [PATCH 9/9] Updated caching to consider the retrofit_key entry when matching the cache --- pyincore/dfr3service.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyincore/dfr3service.py b/pyincore/dfr3service.py index 4e37e4b08..68480b24f 100644 --- a/pyincore/dfr3service.py +++ b/pyincore/dfr3service.py @@ -214,7 +214,7 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: Opt # if retrofit key exist, use retrofit key otherwise use default key retrofit_entry_key = inventory["properties"]["retrofit_k"] if "retrofit_k" in \ inventory["properties"] else None - + cached_curve = self._check_cache(dfr3_sets_cache, inventory["properties"]) if cached_curve is not None: @@ -234,6 +234,9 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: Opt dfr3_sets[inventory['id']] = curve matched_properties_dict = self._convert_properties_to_dict(m.rules, inventory["properties"]) + + if retrofit_entry_key is not None: + matched_properties_dict["retrofit_k"] = retrofit_entry_key # Add the matched inventory properties so other matching inventory can avoid rule matching dfr3_sets_cache[curve] = matched_properties_dict @@ -364,12 +367,16 @@ def _check_cache(dfr3_sets_dict, properties): if not dfr3_sets_dict: return None + retrofit_entry_key = properties["retrofit_k"] if "retrofit_k" in properties else None for entry_key in dfr3_sets_dict: inventory_dict = {} entry_dict = dfr3_sets_dict[entry_key] for rule_key in entry_dict: inventory_dict[rule_key] = properties[rule_key] + if retrofit_entry_key is not None: + inventory_dict["retrofit_k"] = retrofit_entry_key + if entry_dict == inventory_dict: return entry_key