From 8da2b2829903c63565d9317e5dfa46d79ed495c2 Mon Sep 17 00:00:00 2001 From: Tanzim Farhan <52883108+tanzim10@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:35:43 +0600 Subject: [PATCH] black format all the code (#16) * Fixed PR changes for params_regression * Fixed Unittest for param_regression according to PR change * Fixed the mobility notebook and radp_library file(PR changes done) * Resolved test_param_regression unittest cases * Updated Param Regression and radp library as instructed in the PR review * Changed the scipy dependency to solve dependency error * Changed how seed value works especially made it user friendly * Refactored functions and changed the imports * Resolved test_param_regression imports * All Black Changes --- .../cco_engine.py | 59 ++++-- .../cco_example_app.py | 4 +- .../dgpco_cco.py | 29 ++- .../tests/test_cco_engine.py | 23 ++- apps/energy_savings/energy_savings_gym.py | 43 +++-- apps/example/example_app.py | 8 +- radp/client/client.py | 32 +++- radp/client/helper.py | 14 +- radp/common/constants.py | 4 +- radp/common/helpers/file_system_helper.py | 87 ++++++--- .../tests/helpers/test_file_system_helper.py | 132 +++++++++---- radp/digital_twin/mobility/mobility.py | 63 +++++-- .../digital_twin/mobility/param_regression.py | 17 +- .../tests/test_ue_tracks_generation_helper.py | 1 - .../mobility/tests/test_ue_tracks_params.py | 2 +- radp/digital_twin/mobility/ue_tracks.py | 19 +- .../rf/bayesian/bayesian_engine.py | 109 ++++++++--- .../rf/bayesian/tests/test_bayesian_engine.py | 47 +++-- radp/digital_twin/utils/cell_selection.py | 22 ++- radp/digital_twin/utils/gis_tools.py | 74 ++++++-- .../utils/tests/test_cell_selection.py | 33 +++- .../utils/tests/test_gis_tools.py | 2 - radp/example_bayesian_engine_driver_script.py | 33 +++- radp/utility/kafka_utils.py | 16 +- radp/utility/pandas_utils.py | 4 +- radp/utility/simulation_utils.py | 1 + services/api_manager/app.py | 26 ++- .../exceptions/base_api_exception.py | 1 + .../consume_simulation_output_handler.py | 12 +- .../handlers/describe_model_handler.py | 4 +- .../handlers/describe_simulation_handler.py | 8 +- .../handlers/simulation_handler.py | 12 +- .../api_manager/handlers/train_handler.py | 17 +- .../simulation_request_preprocessor.py | 77 +++++--- .../test_consume_simulation_output_handler.py | 44 +++-- .../handlers/test_describe_model_handler.py | 12 +- .../test_describe_simulation_handler.py | 30 ++- .../tests/handlers/test_simulation_handler.py | 13 +- .../tests/handlers/test_train_handler.py | 16 +- .../test_simulation_request_preprocessor.py | 6 +- .../test_simulation_request_validator.py | 12 +- services/api_manager/utils/file_io.py | 12 +- .../simulation_request_validator.py | 7 +- .../orchestration/orchestration_consumer.py | 21 ++- .../orchestration/orchestration_helper.py | 30 ++- services/orchestration/orchestrator.py | 178 +++++++++++++----- .../tests/test_orchestration_helper.py | 54 +++++- .../rf_prediction/rf_prediction_consumer.py | 17 +- .../rf_prediction/rf_prediction_driver.py | 40 ++-- services/training/training_consumer.py | 16 +- services/training/training_driver.py | 18 +- services/ue_tracks_generation/main.py | 4 +- .../ue_tracks_generation_consumer.py | 25 ++- .../ue_tracks_generation_driver.py | 54 ++++-- tests/run_component_tests.py | 20 +- tests/run_end_to_end_tests.py | 16 +- 56 files changed, 1232 insertions(+), 448 deletions(-) diff --git a/apps/coverage_capacity_optimization/cco_engine.py b/apps/coverage_capacity_optimization/cco_engine.py index 84b2fe2..d2b09b3 100755 --- a/apps/coverage_capacity_optimization/cco_engine.py +++ b/apps/coverage_capacity_optimization/cco_engine.py @@ -39,7 +39,6 @@ def rf_to_coverage_dataframe( over_coverage_threshold: float = 0, growth_rate: float = 1, ) -> pd.DataFrame: - if lambda_ <= 0 or lambda_ >= 1: raise ValueError("lambda_ must be between 0 and 1 (noninclusive)") @@ -65,13 +64,20 @@ def rf_to_coverage_dataframe( coverage_dataframe["weak_coverage"] = np.minimum(0, h) coverage_dataframe["overly_covered"] = (h > 0) & (g <= 0) coverage_dataframe["over_coverage"] = np.minimum(0, g) - coverage_dataframe["covered"] = ~coverage_dataframe["weakly_covered"] & ~coverage_dataframe["overly_covered"] + coverage_dataframe["covered"] = ( + ~coverage_dataframe["weakly_covered"] + & ~coverage_dataframe["overly_covered"] + ) # TODO : deprecate the below notion # soft_weak_coverage = sigmoid(h, growth_rate) # soft_over_coverage = sigmoid(g, growth_rate) - coverage_dataframe["soft_weak_coverage"] = 1000 * np.tanh(0.05 * growth_rate * h) - coverage_dataframe["soft_over_coverage"] = 1000 * np.tanh(0.05 * growth_rate * g) + coverage_dataframe["soft_weak_coverage"] = 1000 * np.tanh( + 0.05 * growth_rate * h + ) + coverage_dataframe["soft_over_coverage"] = 1000 * np.tanh( + 0.05 * growth_rate * g + ) coverage_dataframe["network_coverage_utility"] = ( lambda_ * coverage_dataframe["soft_weak_coverage"] + (1 - lambda_) * coverage_dataframe["soft_over_coverage"] @@ -83,8 +89,12 @@ def get_weak_over_coverage_percentages( coverage_dataframe: pd.DataFrame, ) -> Tuple[float, float]: n_points = len(coverage_dataframe.index) - weak_coverage_percent = 100 * coverage_dataframe["weakly_covered"].sum() / n_points - over_coverage_percent = 100 * coverage_dataframe["overly_covered"].sum() / n_points + weak_coverage_percent = ( + 100 * coverage_dataframe["weakly_covered"].sum() / n_points + ) + over_coverage_percent = ( + 100 * coverage_dataframe["overly_covered"].sum() / n_points + ) return weak_coverage_percent, over_coverage_percent @staticmethod @@ -122,18 +132,26 @@ def get_cco_objective_value( coverage_dataframe, ) ) - augmented_coverage_df_with_normalized_traffic_model["network_coverage_utility"] = ( - augmented_coverage_df_with_normalized_traffic_model["normalized_traffic_statistic"] + augmented_coverage_df_with_normalized_traffic_model[ + "network_coverage_utility" + ] = ( + augmented_coverage_df_with_normalized_traffic_model[ + "normalized_traffic_statistic" + ] * coverage_dataframe["network_coverage_utility"] ) - coverage_dataframe["network_coverage_utility"] = augmented_coverage_df_with_normalized_traffic_model[ + coverage_dataframe[ + "network_coverage_utility" + ] = augmented_coverage_df_with_normalized_traffic_model[ "network_coverage_utility" ] if active_ids_list is None: return -math.inf - active_df = coverage_dataframe[coverage_dataframe[id_field].isin(active_ids_list)] + active_df = coverage_dataframe[ + coverage_dataframe[id_field].isin(active_ids_list) + ] active_sector_metric = active_df.groupby(id_field)["network_coverage_utility"] if cco_metric == CcoMetric.PIXEL: @@ -161,7 +179,9 @@ def add_tile_x_and_tile_y( Dataframe with tile_x and tile_y columns appended """ - tile_coords = list(zip(coverage_dataframe[loc_x_field], coverage_dataframe[loc_y_field])) + tile_coords = list( + zip(coverage_dataframe[loc_x_field], coverage_dataframe[loc_y_field]) + ) coverage_dataframe["tile_x"], coverage_dataframe["tile_y"] = zip( *map( @@ -200,11 +220,16 @@ def augment_coverage_df_with_normalized_traffic_model( "over_coverage", """ - sum_of_desired_traffic_statistic_across_all_tiles = traffic_model_df[desired_traffic_statistic_col].sum() + sum_of_desired_traffic_statistic_across_all_tiles = traffic_model_df[ + desired_traffic_statistic_col + ].sum() traffic_model_df["normalized_traffic_statistic"] = ( - traffic_model_df[desired_traffic_statistic_col] / sum_of_desired_traffic_statistic_across_all_tiles + traffic_model_df[desired_traffic_statistic_col] + / sum_of_desired_traffic_statistic_across_all_tiles + ) + coverage_dataframe_with_bing_tiles = CcoEngine.add_tile_x_and_tile_y( + coverage_df ) - coverage_dataframe_with_bing_tiles = CcoEngine.add_tile_x_and_tile_y(coverage_df) augmented_coverage_df_with_normalized_traffic_model = pd.merge( traffic_model_df, coverage_dataframe_with_bing_tiles, @@ -235,6 +260,8 @@ def traffic_normalized_cco_metric(coverage_dataframe: pd.DataFrame) -> float: # only one of weak_coverage and over_coverage can be simultaneously 1 # so, the logic below does not double count return ( - coverage_dataframe["normalized_traffic_statistic"] * coverage_dataframe["weak_coverage"] - + coverage_dataframe["normalized_traffic_statistic"] * coverage_dataframe["over_coverage"] + coverage_dataframe["normalized_traffic_statistic"] + * coverage_dataframe["weak_coverage"] + + coverage_dataframe["normalized_traffic_statistic"] + * coverage_dataframe["over_coverage"] ).sum() diff --git a/apps/coverage_capacity_optimization/cco_example_app.py b/apps/coverage_capacity_optimization/cco_example_app.py index 137e7cc..c2f5f77 100644 --- a/apps/coverage_capacity_optimization/cco_example_app.py +++ b/apps/coverage_capacity_optimization/cco_example_app.py @@ -52,7 +52,9 @@ ) # resolve the model status -- this blocking call ensures training is done and model is available for use -model_status: ModelStatus = radp_helper.resolve_model_status(MODEL_ID, wait_interval=3, max_attempts=10, verbose=True) +model_status: ModelStatus = radp_helper.resolve_model_status( + MODEL_ID, wait_interval=3, max_attempts=10, verbose=True +) # handle an exception if one occurred if not model_status.success: diff --git a/apps/coverage_capacity_optimization/dgpco_cco.py b/apps/coverage_capacity_optimization/dgpco_cco.py index d655b8b..8568f69 100644 --- a/apps/coverage_capacity_optimization/dgpco_cco.py +++ b/apps/coverage_capacity_optimization/dgpco_cco.py @@ -137,7 +137,11 @@ def _single_step( """Single step of DGPCO.""" # calculate new metric - (current_rf_dataframe, current_coverage_dataframe, current_cco_objective,) = self._calc_metric( + ( + current_rf_dataframe, + current_coverage_dataframe, + current_cco_objective, + ) = self._calc_metric( lambda_=lambda_, weak_coverage_threshold=weak_coverage_threshold, over_coverage_threshold=over_coverage_threshold, @@ -151,8 +155,12 @@ def _single_step( # pull the cell config index cell_config_index = self.config.index[self.config["cell_id"] == cell_id][0] - orig_el_idx = self.valid_configuration_values[constants.CELL_EL_DEG].index(orig_el_deg) - cur_el_idx = self.valid_configuration_values[constants.CELL_EL_DEG].index(cur_el_deg) + orig_el_idx = self.valid_configuration_values[constants.CELL_EL_DEG].index( + orig_el_deg + ) + cur_el_idx = self.valid_configuration_values[constants.CELL_EL_DEG].index( + cur_el_deg + ) for d in opt_delta: new_el_idx = orig_el_idx + d @@ -161,11 +169,15 @@ def _single_step( # we do not want to check current value continue - if new_el_idx < 0 or new_el_idx >= len(self.valid_configuration_values[constants.CELL_EL_DEG]): + if new_el_idx < 0 or new_el_idx >= len( + self.valid_configuration_values[constants.CELL_EL_DEG] + ): # we do not want to wrap around, since that would not be a neighboring tilt continue - new_el = self.valid_configuration_values[constants.CELL_EL_DEG][new_el_idx] + new_el = self.valid_configuration_values[constants.CELL_EL_DEG][ + new_el_idx + ] # update the cell config el_degree self.config.loc[cell_config_index, constants.CELL_EL_DEG] = new_el @@ -258,7 +270,12 @@ def _single_step( logging.info(f"\nIn epoch: {epoch:02}/{num_epochs}...") # Perform one step of DGPCO - (new_opt_el, new_rf_dataframe, new_coverage_dataframe, new_cco_objective_value,) = _single_step( + ( + new_opt_el, + new_rf_dataframe, + new_coverage_dataframe, + new_cco_objective_value, + ) = _single_step( cell_id=cell_id, orig_el_deg=orig_el_deg, cur_el_deg=cur_el_deg, diff --git a/apps/coverage_capacity_optimization/tests/test_cco_engine.py b/apps/coverage_capacity_optimization/tests/test_cco_engine.py index dc9f536..191d86d 100644 --- a/apps/coverage_capacity_optimization/tests/test_cco_engine.py +++ b/apps/coverage_capacity_optimization/tests/test_cco_engine.py @@ -14,7 +14,9 @@ class TestCCO(unittest.TestCase): @classmethod def setUpClass(cls): - cls.dummy_df = pd.DataFrame(data={CELL_ID: [1, 2, 73], LOC_X: [3, 4, 89], LOC_Y: [7, 8, 10]}) + cls.dummy_df = pd.DataFrame( + data={CELL_ID: [1, 2, 73], LOC_X: [3, 4, 89], LOC_Y: [7, 8, 10]} + ) def test_invalid_lambda(self): self.dummy_df["rsrp_dbm"] = [98, 92, 86] @@ -37,7 +39,8 @@ def testing_weakly_covered(self): self.dummy_df, weak_coverage_threshold=-100, over_coverage_threshold=0 ) self.assertEqual( - returned_df["weakly_covered"][returned_df["weakly_covered"] == 1].count() == 1, + returned_df["weakly_covered"][returned_df["weakly_covered"] == 1].count() + == 1, True, ) @@ -58,7 +61,8 @@ def test_overly_covered(self): self.dummy_df, weak_coverage_threshold=-100, over_coverage_threshold=0 ) self.assertEqual( - returned_df["overly_covered"][returned_df["overly_covered"] == 0].count() == 1, + returned_df["overly_covered"][returned_df["overly_covered"] == 0].count() + == 1, True, ) @@ -81,11 +85,13 @@ def testing_some_not_weakly_or_overcovered(self): True, ) self.assertEqual( - returned_df["weakly_covered"][returned_df["weakly_covered"] == 1].count() == 1, + returned_df["weakly_covered"][returned_df["weakly_covered"] == 1].count() + == 1, True, ) self.assertEqual( - returned_df["overly_covered"][returned_df["overly_covered"] == 1].count() == 1, + returned_df["overly_covered"][returned_df["overly_covered"] == 1].count() + == 1, True, ) @@ -168,11 +174,14 @@ def test_get_cco_objective_value(self): ) # asserting the multiplied version of network_coverage_utility self.assertTrue( - coverage_df["network_coverage_utility"][0] == 0.24 and coverage_df["network_coverage_utility"][1] == 0.24 + coverage_df["network_coverage_utility"][0] == 0.24 + and coverage_df["network_coverage_utility"][1] == 0.24 ) # asserting the average of network_coverage_utility - self.assertTrue(0.24 == coverage_df["network_coverage_utility"].sum() / len(coverage_df)) + self.assertTrue( + 0.24 == coverage_df["network_coverage_utility"].sum() / len(coverage_df) + ) # asserting the returned cco_objective_value expected_value = 0.24 diff --git a/apps/energy_savings/energy_savings_gym.py b/apps/energy_savings/energy_savings_gym.py index 82b1f6c..c18c244 100644 --- a/apps/energy_savings/energy_savings_gym.py +++ b/apps/energy_savings/energy_savings_gym.py @@ -98,7 +98,9 @@ def __init__( self.prediction_dfs = dict() for cell_id in site_config_df.cell_id: prediction_dfs = BayesianDigitalTwin.create_prediction_frames( - site_config_df=self.site_config_df[self.site_config_df.cell_id.isin([cell_id])].reset_index(), + site_config_df=self.site_config_df[ + self.site_config_df.cell_id.isin([cell_id]) + ].reset_index(), prediction_frame_template=prediction_frame_template[cell_id], ) self.prediction_dfs.update(prediction_dfs) @@ -140,11 +142,13 @@ def __init__( # Reward when all cells are off: self.r_norm = (1 - lambda_) * ( - -10 * np.log10(self.num_cells) - over_coverage_threshold + min_rsrp - weak_coverage_threshold + -10 * np.log10(self.num_cells) + - over_coverage_threshold + + min_rsrp + - weak_coverage_threshold ) def _next_observation(self): - if self.ue_tracks: data = next(self.ue_tracks) for batch in data: @@ -198,17 +202,22 @@ def _next_observation(self): ) if self.traffic_model_df is None: cco_objective_metric = ( - coverage_dataframe["weak_coverage"].mean() + coverage_dataframe["over_coverage"].mean() + coverage_dataframe["weak_coverage"].mean() + + coverage_dataframe["over_coverage"].mean() ) else: - processed_coverage_dataframe = CcoEngine.augment_coverage_df_with_normalized_traffic_model( - self.traffic_model_df, - "avg_of_average_egress_kbps_across_all_time", - coverage_dataframe, + processed_coverage_dataframe = ( + CcoEngine.augment_coverage_df_with_normalized_traffic_model( + self.traffic_model_df, + "avg_of_average_egress_kbps_across_all_time", + coverage_dataframe, + ) ) - cco_objective_metric = CcoEngine.traffic_normalized_cco_metric(processed_coverage_dataframe) + cco_objective_metric = CcoEngine.traffic_normalized_cco_metric( + processed_coverage_dataframe + ) # Output for debugging/postprocessing purposes if self.debug: @@ -216,7 +225,9 @@ def _next_observation(self): self.coverage_dataframe = coverage_dataframe return ( - EnergySavingsGym.ENERGY_MAX_PER_CELL * sum(self.on_off_state) / len(self.on_off_state), + EnergySavingsGym.ENERGY_MAX_PER_CELL + * sum(self.on_off_state) + / len(self.on_off_state), 0.0, cco_objective_metric, # TODO : normalized this against MAX_CLUSTER_CCO ) @@ -230,7 +241,11 @@ def reward( if energy_consumption == 0: return self.r_norm else: - return self.lambda_ * -1.0 * energy_consumption + (1 - self.lambda_) * cco_objective_metric - self.r_norm + return ( + self.lambda_ * -1.0 * energy_consumption + + (1 - self.lambda_) * cco_objective_metric + - self.r_norm + ) def make_action_from_state(self): action = np.empty(self.num_cells, dtype=int) @@ -242,7 +257,6 @@ def make_action_from_state(self): return action def _take_action(self, action): - num_cells = len(self.site_config_df) # on_off_cell_state captures the on/off state of each cell (on is `1`) on_off_cell_state = [1] * num_cells @@ -277,7 +291,6 @@ def reset(self): return self._next_observation() def step(self, action): - # Execute one time step within the environment self._take_action(action) @@ -293,7 +306,9 @@ def step(self, action): return obs, reward, done, {} - def get_all_possible_actions(self, possible_actions: List[List[int]]) -> List[List[int]]: + def get_all_possible_actions( + self, possible_actions: List[List[int]] + ) -> List[List[int]]: """ A recursive function to get all possible actions as a list. Useful for bruteforce search. diff --git a/apps/example/example_app.py b/apps/example/example_app.py index 3ef0d4b..e116e05 100644 --- a/apps/example/example_app.py +++ b/apps/example/example_app.py @@ -136,9 +136,7 @@ train_response = radp_client.train( model_id=MODEL_ID, params=TRAINING_PARAMS, - ue_training_data=pd.concat( - [pd.read_csv(file) for file in TRAINING_DATA_FILES] - ), + ue_training_data=pd.concat([pd.read_csv(file) for file in TRAINING_DATA_FILES]), topology=pd.read_csv(TOPOLOGY_FILE), ) @@ -164,9 +162,7 @@ # run simulation on cumulative data passed to model simulation_response = radp_client.simulation( simulation_event=simulation_event, - ue_data=pd.concat( - [pd.read_csv(file) for file in PREDICTION_DATA_FILES] - ), + ue_data=pd.concat([pd.read_csv(file) for file in PREDICTION_DATA_FILES]), config=pd.read_csv(PREDICTION_CONFIG), ) simulation_id = simulation_response["simulation_id"] diff --git a/radp/client/client.py b/radp/client/client.py index 0a00878..5608a5c 100644 --- a/radp/client/client.py +++ b/radp/client/client.py @@ -62,7 +62,9 @@ def describe_model(self, model_id: str) -> Dict: # TODO: add error handling logic for when something goes wrong @retry(exceptions=RETRY_EXCEPTIONS, tries=3, delay=1, backoff=2) def describe_simulation(self, simulation_id: str) -> Dict: - logger.debug(f"Calling describe_simulation api with simulation: '{simulation_id}'") + logger.debug( + f"Calling describe_simulation api with simulation: '{simulation_id}'" + ) path = f"{constants.DESCRIBE_SIMULATION_API_PATH}/{simulation_id}" response = self._send_get_request(path, {}) @@ -101,7 +103,9 @@ def train( payload_file = json.dumps(payload) # open user provided training csv files - with open(ue_training_data, "r") as ue_training_data_file, open(topology, "r") as topology_file: + with open(ue_training_data, "r") as ue_training_data_file, open( + topology, "r" + ) as topology_file: # send a json body file as well as both csv files files: Set[Any] = { ( @@ -166,7 +170,9 @@ def simulation( } if not ue_data and not config: - return self._send_post_request(constants.SIMULATION_API_PATH, files=files).json() + return self._send_post_request( + constants.SIMULATION_API_PATH, files=files + ).json() if not config: with open(str(ue_data), "r") as ue_data_file: @@ -180,7 +186,9 @@ def simulation( ), ) ) - return self._send_post_request(constants.SIMULATION_API_PATH, files=files).json() + return self._send_post_request( + constants.SIMULATION_API_PATH, files=files + ).json() if not ue_data: with open(str(config), "r") as config_file: @@ -194,7 +202,9 @@ def simulation( ), ) ) - return self._send_post_request(constants.SIMULATION_API_PATH, files=files).json() + return self._send_post_request( + constants.SIMULATION_API_PATH, files=files + ).json() with open(ue_data, "r") as ue_data_file, open(config, "r") as config_file: files.add( @@ -217,17 +227,23 @@ def simulation( ), ), ) - return self._send_post_request(constants.SIMULATION_API_PATH, files=files).json() + return self._send_post_request( + constants.SIMULATION_API_PATH, files=files + ).json() # TODO: add error handling logic for when something goes wrong @retry(exceptions=RETRY_EXCEPTIONS, tries=3, delay=1, backoff=2) def consume_simulation_output(self, simulation_id: str) -> pd.DataFrame: - logger.debug(f"Calling consume_simulation_output api with simulation: '{simulation_id}'") + logger.debug( + f"Calling consume_simulation_output api with simulation: '{simulation_id}'" + ) path = f"{constants.CONSUME_SIMULATION_OUTPUT_API_PATH}/{simulation_id}/{constants.DOWNLOAD}" consume_simulation_output_url = self._get_request_url(path, {}) # TODO: this only works for a single file in zipfile # we will need to update this once batching is supported - rf_dataframe = pd.read_csv(consume_simulation_output_url, compression=constants.ZIP_COMPRESSION) + rf_dataframe = pd.read_csv( + consume_simulation_output_url, compression=constants.ZIP_COMPRESSION + ) return rf_dataframe diff --git a/radp/client/helper.py b/radp/client/helper.py index 456a175..0b94505 100644 --- a/radp/client/helper.py +++ b/radp/client/helper.py @@ -53,7 +53,10 @@ def resolve_model_status( describe_model_response = self.radp_client.describe_model(model_id) if not describe_model_response[constants.MODEL_EXISTS]: print_if_verbose("Model not yet created", verbose) - elif describe_model_response[constants.MODEL_STATUS] != constants.MODEL_TRAINED: + elif ( + describe_model_response[constants.MODEL_STATUS] + != constants.MODEL_TRAINED + ): print_if_verbose("Model not yet trained", verbose) elif job_id and describe_model_response[constants.JOB_ID] != job_id: print_if_verbose("Model training job not yet complete", verbose) @@ -79,10 +82,15 @@ def resolve_simulation_status( """Resolve the status of a RADP simulation""" attempt = 0 while attempt < max_attempts: - describe_simulation_response = self.radp_client.describe_simulation(simulation_id) + describe_simulation_response = self.radp_client.describe_simulation( + simulation_id + ) if not describe_simulation_response[constants.SIMULATION_EXISTS]: print_if_verbose("Simulation not yet created", verbose) - elif describe_simulation_response[constants.SIMULATION_STATUS] != constants.SIMULATION_FINISHED: + elif ( + describe_simulation_response[constants.SIMULATION_STATUS] + != constants.SIMULATION_FINISHED + ): print_if_verbose("Simulation not yet finished", verbose) elif job_id and describe_simulation_response[constants.JOB_ID] != job_id: print_if_verbose("Simulation job not yet complete", verbose) diff --git a/radp/common/constants.py b/radp/common/constants.py index 508a647..a44ec93 100644 --- a/radp/common/constants.py +++ b/radp/common/constants.py @@ -118,7 +118,9 @@ RNG_SEED = "rng_seed" LON_X_DIMS = "lon_x_dims" LON_Y_DIMS = "lon_y_dims" -UE_TRACK_GENERATION_OUTPUTS_FOLDER = "/srv/radp/simulation_data/outputs/ue_tracks_generation" +UE_TRACK_GENERATION_OUTPUTS_FOLDER = ( + "/srv/radp/simulation_data/outputs/ue_tracks_generation" +) GAUSS_MARKOV_PARAMS = "gauss_markov_params" # Protocol Emulation related diff --git a/radp/common/helpers/file_system_helper.py b/radp/common/helpers/file_system_helper.py index 2c8b888..b3508e1 100644 --- a/radp/common/helpers/file_system_helper.py +++ b/radp/common/helpers/file_system_helper.py @@ -40,7 +40,9 @@ def gen_simulation_metadata_file_path(simulation_id: str) -> str: @staticmethod def gen_simulation_ue_data_file_path(simulation_id: str) -> str: """Helper method to generated simulation ue data file path""" - simulation_directory = RADPFileSystemHelper.gen_simulation_directory(simulation_id) + simulation_directory = RADPFileSystemHelper.gen_simulation_directory( + simulation_id + ) return os.path.join( simulation_directory, f"{constants.UE_DATA_FILE_NAME}.{constants.DF_FILE_EXTENSION}", @@ -49,7 +51,9 @@ def gen_simulation_ue_data_file_path(simulation_id: str) -> str: @staticmethod def gen_simulation_cell_config_file_path(simulation_id: str) -> str: """Helper method to generated simulation config file path""" - simulation_directory = RADPFileSystemHelper.gen_simulation_directory(simulation_id) + simulation_directory = RADPFileSystemHelper.gen_simulation_directory( + simulation_id + ) return os.path.join( simulation_directory, f"{constants.CONFIG_FILE_NAME}.{constants.DF_FILE_EXTENSION}", @@ -58,24 +62,32 @@ def gen_simulation_cell_config_file_path(simulation_id: str) -> str: @staticmethod def load_simulation_metadata(simulation_id: str) -> Dict: """Helper method to load simulation metadata to an object""" - metadata_file_path = RADPFileSystemHelper.gen_simulation_metadata_file_path(simulation_id) + metadata_file_path = RADPFileSystemHelper.gen_simulation_metadata_file_path( + simulation_id + ) try: with open(metadata_file_path, "r") as json_file: return json.load(json_file) except Exception as e: - logger.exception(f"Exception occurred while loading metadata for simulation: {simulation_id}: {e}") + logger.exception( + f"Exception occurred while loading metadata for simulation: {simulation_id}: {e}" + ) raise e @staticmethod def save_simulation_metadata(sim_metadata: Dict, simulation_id: str): """Helper method to save simulation metadata to file""" - metadata_file_path = RADPFileSystemHelper.gen_simulation_metadata_file_path(simulation_id) + metadata_file_path = RADPFileSystemHelper.gen_simulation_metadata_file_path( + simulation_id + ) try: with atomic_write(metadata_file_path, "w") as json_file: json.dump(sim_metadata, json_file) except Exception as e: - logger.exception(f"Exception occurred while saving metadata for simulation: {simulation_id}: {e}") + logger.exception( + f"Exception occurred while saving metadata for simulation: {simulation_id}: {e}" + ) raise e @staticmethod @@ -84,7 +96,9 @@ def save_simulation_ue_data(simulation_id: str, ue_data_file_path: str): ue_data_file_path - file path of ue data csv file passed in by user """ - sim_ue_data_file_path = RADPFileSystemHelper.gen_simulation_ue_data_file_path(simulation_id) + sim_ue_data_file_path = RADPFileSystemHelper.gen_simulation_ue_data_file_path( + simulation_id + ) try: # load UE data df and save to feather format with open(ue_data_file_path, "r") as csv_file: @@ -101,7 +115,9 @@ def save_simulation_cell_config(simulation_id: str, config_file_path: str): config_file_path - file path of config csv file passed in by user """ - sim_config_file_path = RADPFileSystemHelper.gen_simulation_cell_config_file_path(simulation_id) + sim_config_file_path = ( + RADPFileSystemHelper.gen_simulation_cell_config_file_path(simulation_id) + ) try: # load config df and save to feather format @@ -128,7 +144,9 @@ def hash_val_found_in_output_folder(stage: SimulationStage, hash_val: str) -> bo return False @staticmethod - def gen_stage_output_file_path(stage: SimulationStage, hash_val: str, batch: int) -> str: + def gen_stage_output_file_path( + stage: SimulationStage, hash_val: str, batch: int + ) -> str: """Helper method to generate a file path for a specific stage output""" stage_output_folder = os.path.join( constants.SIMULATION_DATA_FOLDER, @@ -142,8 +160,12 @@ def gen_stage_output_file_path(stage: SimulationStage, hash_val: str, batch: int def gen_sim_output_zip_file_path(simulation_id: str, include_ext=True): """Generate the zip file path for a given simulation""" zip_file_name = f"{simulation_id}-{constants.SIM_OUTPUT_FILE_SUFFIX}" - zip_file_name = zip_file_name + (f".{constants.SIM_OUTPUT_FILE_EXTENSION}" if include_ext else "") - return os.path.join(constants.SIMULATION_DATA_FOLDER, simulation_id, zip_file_name) + zip_file_name = zip_file_name + ( + f".{constants.SIM_OUTPUT_FILE_EXTENSION}" if include_ext else "" + ) + return os.path.join( + constants.SIMULATION_DATA_FOLDER, simulation_id, zip_file_name + ) @staticmethod def gen_sim_output_directory(simulation_id: str): @@ -167,7 +189,9 @@ def zip_output_files_to_simulation_folder_as_csvs( stage_output_file_paths = RADPFileSystemHelper.get_stage_output_file_paths( stage=stage, hash_val=hash_val, num_batches=num_batches ) - simulation_output_directory = RADPFileSystemHelper.gen_sim_output_directory(simulation_id) + simulation_output_directory = RADPFileSystemHelper.gen_sim_output_directory( + simulation_id + ) # create output directory if it does not already exist if not os.path.exists(simulation_output_directory): @@ -187,11 +211,15 @@ def zip_output_files_to_simulation_folder_as_csvs( output_df = read_feather_df(fp) output_df.to_csv(new_file_path, index=False) except Exception as e: - logger.exception("Exception occurred while writing csv's to output folder") + logger.exception( + "Exception occurred while writing csv's to output folder" + ) raise e # get the zip file path - zip_file_path = RADPFileSystemHelper.gen_sim_output_zip_file_path(simulation_id, include_ext=False) + zip_file_path = RADPFileSystemHelper.gen_sim_output_zip_file_path( + simulation_id, include_ext=False + ) try: # chdir to simulation output directory to only zip files @@ -203,11 +231,15 @@ def zip_output_files_to_simulation_folder_as_csvs( ) logger.info(f"Zipped output files to {zip_file_path}") except Exception as e: - logger.exception(f"Exception occurred zipping files in simulation: {simulation_id}: {e}") + logger.exception( + f"Exception occurred zipping files in simulation: {simulation_id}: {e}" + ) raise e @staticmethod - def get_stage_output_file_paths(stage: SimulationStage, hash_val: str, num_batches: int) -> List[str]: + def get_stage_output_file_paths( + stage: SimulationStage, hash_val: str, num_batches: int + ) -> List[str]: """Helper method to get list of output files for a stage""" # get output folder stage_output_folder = os.path.join( @@ -220,11 +252,14 @@ def get_stage_output_file_paths(stage: SimulationStage, hash_val: str, num_batch file_name = f"{stage.value}-{hash_val}" file_path_without_batch = os.path.join(stage_output_folder, file_name) return [ - f"{file_path_without_batch}-{batch}.{constants.DF_FILE_EXTENSION}" for batch in range(1, num_batches + 1) + f"{file_path_without_batch}-{batch}.{constants.DF_FILE_EXTENSION}" + for batch in range(1, num_batches + 1) ] @staticmethod - def clear_output_data_from_stage(stage: SimulationStage, save_hash_val: Optional[str]): + def clear_output_data_from_stage( + stage: SimulationStage, save_hash_val: Optional[str] + ): """ Clear the output from a stage unless it contains the save hash value @@ -256,9 +291,13 @@ def clear_output_data_from_stage(stage: SimulationStage, save_hash_val: Optional file_path = os.path.join(stage_output_folder, file_name) os.remove(file_path) delete_count += 1 - logger.info(f"Cleared {delete_count} unused outputs from stage: {stage.value}") + logger.info( + f"Cleared {delete_count} unused outputs from stage: {stage.value}" + ) except Exception as e: - logger.exception(f"Exception occurred while deleting outputs in stage: {stage.value}") + logger.exception( + f"Exception occurred while deleting outputs in stage: {stage.value}" + ) raise e @staticmethod @@ -324,7 +363,9 @@ def load_model_metadata(model_id: str) -> Dict: logger.debug(f"Loaded metadata for model: {model_id}") return metadata except Exception as e: - logger.exception(f"Exception occurred loading metadata for model: {model_id}") + logger.exception( + f"Exception occurred loading metadata for model: {model_id}" + ) raise e @staticmethod @@ -345,7 +386,9 @@ def save_model_metadata(model_id: str, model_metadata: Dict): json.dump(model_metadata, json_file) logger.debug(f"Saved metadata for model: {model_id}") except Exception as e: - logger.exception(f"Exception occurred loading metadata for model: {model_id}") + logger.exception( + f"Exception occurred loading metadata for model: {model_id}" + ) raise e @staticmethod diff --git a/radp/common/tests/helpers/test_file_system_helper.py b/radp/common/tests/helpers/test_file_system_helper.py index d6ddf6c..6b82e54 100644 --- a/radp/common/tests/helpers/test_file_system_helper.py +++ b/radp/common/tests/helpers/test_file_system_helper.py @@ -20,8 +20,12 @@ class TestRADPFileSystemHelper(TestCase): "/dummy_simulation_data_folder_path", ) def test_gen_simulation_directory(self): - simulation_directory = RADPFileSystemHelper.gen_simulation_directory(simulation_id="dummy_simulation") - self.assertEqual(simulation_directory, "/dummy_simulation_data_folder_path/dummy_simulation") + simulation_directory = RADPFileSystemHelper.gen_simulation_directory( + simulation_id="dummy_simulation" + ) + self.assertEqual( + simulation_directory, "/dummy_simulation_data_folder_path/dummy_simulation" + ) @patch( "radp.common.helpers.file_system_helper.constants.SIMULATION_DATA_FOLDER", @@ -58,8 +62,10 @@ def test_gen_simulation_metadata_file_path(self): "dummy_df_file_extension", ) def test_gen_simulation_ue_data_file_path(self): - simulation_ue_data_file_path = RADPFileSystemHelper.gen_simulation_ue_data_file_path( - simulation_id="dummy_simulation" + simulation_ue_data_file_path = ( + RADPFileSystemHelper.gen_simulation_ue_data_file_path( + simulation_id="dummy_simulation" + ) ) self.assertEqual( simulation_ue_data_file_path, @@ -79,8 +85,10 @@ def test_gen_simulation_ue_data_file_path(self): "dummy_df_file_extension", ) def test_gen_simulation_cell_config_file_path(self): - simulation_cell_config_file_path = RADPFileSystemHelper.gen_simulation_cell_config_file_path( - simulation_id="dummy_simulation" + simulation_cell_config_file_path = ( + RADPFileSystemHelper.gen_simulation_cell_config_file_path( + simulation_id="dummy_simulation" + ) ) self.assertEqual( simulation_cell_config_file_path, @@ -89,7 +97,9 @@ def test_gen_simulation_cell_config_file_path(self): # replace builtins.open with a mocked open operation @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_metadata_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_metadata_file_path" + ) def test_load_simulation_metadata(self, mocked_metadata_file_path): mocked_metadata_file_path.return_value = "dummy_sim_data_file_path" self.assertEqual( @@ -102,7 +112,9 @@ def test_load_simulation_metadata(self, mocked_metadata_file_path): ) @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_metadata_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_metadata_file_path" + ) def test_load_simulation_metadata_exception(self, mocked_metadata_file_path): mocked_metadata_file_path.side_effect = Exception("dummy exception!") with self.assertRaises(Exception) as e: @@ -113,41 +125,61 @@ def test_load_simulation_metadata_exception(self, mocked_metadata_file_path): @patch("radp.common.helpers.file_system_helper.atomic_write") @patch("radp.common.helpers.file_system_helper.json") def test_save_simulation_metadata(self, mocked_json, mocked_atomic_write): - RADPFileSystemHelper.save_simulation_metadata(mocked_sim_data, "dummy_simulation") + RADPFileSystemHelper.save_simulation_metadata( + mocked_sim_data, "dummy_simulation" + ) mocked_atomic_write.assert_called_once() mocked_json.dump.assert_called_once() @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_metadata_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_metadata_file_path" + ) def test_save_simulation_metadata_exception(self, mocked_metadata_file_path): mocked_metadata_file_path.side_effect = Exception("dummy exception!") with self.assertRaises(Exception) as e: - RADPFileSystemHelper.save_simulation_metadata(mocked_sim_data, "dummy_simulation") + RADPFileSystemHelper.save_simulation_metadata( + mocked_sim_data, "dummy_simulation" + ) self.assertEqual(str(e.exception), "dummy exception!") @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) @patch("radp.common.helpers.file_system_helper.pd.read_csv") @patch("radp.common.helpers.file_system_helper.write_feather_df") - def test_save_simulation_ue_data(self, mocked_pandas_read_csv, mocked_write_feather_df): - RADPFileSystemHelper.save_simulation_ue_data("dummy_simulation", "dummy_config_file_path") + def test_save_simulation_ue_data( + self, mocked_pandas_read_csv, mocked_write_feather_df + ): + RADPFileSystemHelper.save_simulation_ue_data( + "dummy_simulation", "dummy_config_file_path" + ) mocked_pandas_read_csv.assert_called_once() mocked_write_feather_df.assert_called_once() @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_ue_data_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_simulation_ue_data_file_path" + ) @patch("radp.common.helpers.file_system_helper.write_feather_df") - def test_save_simulation_ue_data_exception(self, mocked_simulation_ue_data_file_path, mocked_write_feather_df): + def test_save_simulation_ue_data_exception( + self, mocked_simulation_ue_data_file_path, mocked_write_feather_df + ): mocked_simulation_ue_data_file_path.side_effect = Exception("dummy exception!") with self.assertRaises(Exception) as e: - RADPFileSystemHelper.save_simulation_ue_data("dummy_simulation", "dummy_config_file_path") + RADPFileSystemHelper.save_simulation_ue_data( + "dummy_simulation", "dummy_config_file_path" + ) mocked_write_feather_df.assert_called_once() self.assertEqual(str(e.exception), "dummy exception!") @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) @patch("radp.utility.pandas_utils.atomic_write") @patch("radp.common.helpers.file_system_helper.pd.read_csv") - def test_save_simulation_cell_config(self, mocked_pandas_read_csv, mocked_atomic_write): - RADPFileSystemHelper.save_simulation_cell_config("dummy_simulation", "dummy_config_file_path") + def test_save_simulation_cell_config( + self, mocked_pandas_read_csv, mocked_atomic_write + ): + RADPFileSystemHelper.save_simulation_cell_config( + "dummy_simulation", "dummy_config_file_path" + ) mocked_atomic_write.assert_called_once() mocked_pandas_read_csv.assert_called_once() @@ -156,10 +188,16 @@ def test_save_simulation_cell_config(self, mocked_pandas_read_csv, mocked_atomic """radp.common.helpers.file_system_helper.\ RADPFileSystemHelper.gen_simulation_cell_config_file_path""" ) - def test_save_simulation_cell_config_exception(self, mocked_simulation_cell_config_file_path): - mocked_simulation_cell_config_file_path.side_effect = Exception("dummy exception!") + def test_save_simulation_cell_config_exception( + self, mocked_simulation_cell_config_file_path + ): + mocked_simulation_cell_config_file_path.side_effect = Exception( + "dummy exception!" + ) with self.assertRaises(Exception) as e: - RADPFileSystemHelper.save_simulation_cell_config("dummy_simulation", "dummy_config_file_path") + RADPFileSystemHelper.save_simulation_cell_config( + "dummy_simulation", "dummy_config_file_path" + ) self.assertEqual(str(e.exception), "dummy exception!") @patch( @@ -178,7 +216,11 @@ def test_save_simulation_cell_config_exception(self, mocked_simulation_cell_conf def test_hash_val_found_in_output_folder(self, mocked_listdir): mocked_listdir.return_value = ["dummy_dir_1", "dummy_dir_2"] dummy_stage = SimulationStage.UE_TRACKS_GENERATION - self.assertTrue(RADPFileSystemHelper.hash_val_found_in_output_folder(dummy_stage, "dummy_dir")) + self.assertTrue( + RADPFileSystemHelper.hash_val_found_in_output_folder( + dummy_stage, "dummy_dir" + ) + ) @patch( "radp.common.helpers.file_system_helper.constants.SIMULATION_DATA_FOLDER", @@ -200,7 +242,11 @@ def test_hash_val_found_in_output_folder(self, mocked_listdir): def test_hash_val_found_in_output_folder_neg(self, mocked_listdir): mocked_listdir.return_value = ["dummy_dir_1", "dummy_dir_2"] dummy_stage = SimulationStage.UE_TRACKS_GENERATION - self.assertFalse(RADPFileSystemHelper.hash_val_found_in_output_folder(dummy_stage, "dummy_other_str")) + self.assertFalse( + RADPFileSystemHelper.hash_val_found_in_output_folder( + dummy_stage, "dummy_other_str" + ) + ) @patch( "radp.common.helpers.file_system_helper.constants.SIMULATION_DATA_FOLDER", @@ -219,7 +265,9 @@ def test_gen_stage_output_file_path(self): dummy_hash_val = "dummy_hash_val" dummy_batch = 1 self.assertEqual( - RADPFileSystemHelper.gen_stage_output_file_path(dummy_stage, dummy_hash_val, dummy_batch), + RADPFileSystemHelper.gen_stage_output_file_path( + dummy_stage, dummy_hash_val, dummy_batch + ), "/dummy_simulation_data_folder_path/dummy_simulation_outputs_folder/" "ue_tracks_generation/ue_tracks_generation-dummy_hash_val-1.dummy_df_file_extension", ) @@ -253,7 +301,9 @@ def test_gen_sim_output_zip_file_path(self): ) def test_gen_sim_output_zip_file_path_neg(self): self.assertEqual( - RADPFileSystemHelper.gen_sim_output_zip_file_path("dummy_simulation", False), + RADPFileSystemHelper.gen_sim_output_zip_file_path( + "dummy_simulation", False + ), "/dummy_simulation_data_folder_path/dummy_simulation/dummy_simulation-dummy_output_file_suffix", ) @@ -305,7 +355,9 @@ def test_get_stage_output_file_paths(self): @patch("os.listdir") @patch("os.remove") - def test_clear_output_data_from_stage_no_save_hash_val(self, mocked_listdir, mocked_remove): + def test_clear_output_data_from_stage_no_save_hash_val( + self, mocked_listdir, mocked_remove + ): mocked_listdir.return_value = ["dummy_file_1"] dummy_stage = SimulationStage.START RADPFileSystemHelper.clear_output_data_from_stage(dummy_stage, None) @@ -313,7 +365,9 @@ def test_clear_output_data_from_stage_no_save_hash_val(self, mocked_listdir, moc @patch("os.listdir") @patch("os.remove") - def test_clear_output_data_from_stage_with_hash_val(self, mocked_listdir, mocked_remove): + def test_clear_output_data_from_stage_with_hash_val( + self, mocked_listdir, mocked_remove + ): mocked_listdir.return_value = ["dummy_file_1", "dummy_file_2"] dummy_stage = SimulationStage.START RADPFileSystemHelper.clear_output_data_from_stage(dummy_stage, "_2") @@ -429,7 +483,9 @@ def test_gen_model_topology_file_path(self): "/dummy_models_folder/dummy_model_id/dummy_topology_file_name.dummy_df_file_extension", ) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_metadata_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_metadata_file_path" + ) @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) def test_load_model_metadata(self, mocked_model_metadata_file_path): mocked_model_metadata_file_path.return_value = "dummy_model_metadata_file_path" @@ -446,7 +502,9 @@ def test_load_model_metadata(self, mocked_model_metadata_file_path): mocked_sim_data, ) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_metadata_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_metadata_file_path" + ) @patch("builtins.open", mock_open(read_data=json_mocked_sim_data)) def test_load_model_metadata_exception(self, mocked_model_metadata_file_path): mocked_model_metadata_file_path.side_effect = Exception("dummy exception!") @@ -488,7 +546,9 @@ def test_load_model_metadata_exception(self, mocked_model_metadata_file_path): # "dummy exception!", # ) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_file_path" + ) def test_check_model_exists(self, mocked_model_file_path): model_file_path = RADPFileSystemHelper.gen_model_file_path( "dummy_model_id", @@ -500,7 +560,9 @@ def test_check_model_exists(self, mocked_model_file_path): ) ) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_file_path") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.gen_model_file_path" + ) def test_check_model_exists_neg(self, mocked_model_file_path): mocked_model_file_path.side_effect = Exception("dummy exception!") with self.assertRaises(Exception) as e: @@ -512,7 +574,9 @@ def test_check_model_exists_neg(self, mocked_model_file_path): "dummy exception!", ) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.load_model_metadata") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.load_model_metadata" + ) @patch( "radp.common.helpers.file_system_helper.constants.STATUS", "dummy_status", @@ -526,7 +590,9 @@ def test_get_model_status(self, mocked_model_metadata): "trained", ) - @patch("radp.common.helpers.file_system_helper.RADPFileSystemHelper.load_model_metadata") + @patch( + "radp.common.helpers.file_system_helper.RADPFileSystemHelper.load_model_metadata" + ) @patch( "radp.common.helpers.file_system_helper.constants.MODEL_TYPE", "dummy_model_type", diff --git a/radp/digital_twin/mobility/mobility.py b/radp/digital_twin/mobility/mobility.py index 3973a88..964c773 100644 --- a/radp/digital_twin/mobility/mobility.py +++ b/radp/digital_twin/mobility/mobility.py @@ -32,12 +32,16 @@ def U(rng, MIN, MAX, SAMPLES): # define a Truncated Power Law Distribution def P(rng, ALPHA, MIN, MAX, SAMPLES): - return ((MAX ** (ALPHA + 1.0) - 1.0) * rng.random(SAMPLES.shape) + 1.0) ** (1.0 / (ALPHA + 1.0)) + return ((MAX ** (ALPHA + 1.0) - 1.0) * rng.random(SAMPLES.shape) + 1.0) ** ( + 1.0 / (ALPHA + 1.0) + ) # *************** Palm state probability ********************** def pause_probability_init(pause_low, pause_high, speed_low, speed_high, dimensions): - alpha1 = ((pause_high + pause_low) * (speed_high - speed_low)) / (2 * np.log(speed_high / speed_low)) + alpha1 = ((pause_high + pause_low) * (speed_high - speed_low)) / ( + 2 * np.log(speed_high / speed_low) + ) delta1 = np.sqrt(np.sum(np.square(dimensions))) return alpha1 / (alpha1 + delta1) @@ -51,7 +55,9 @@ def residual_time(rng, mean, delta, shape=(1,)): if delta != 0.0: case_1_u = u < (2.0 * t1 / (t1 + t2)) residual[case_1_u] = u[case_1_u] * (t1 + t2) / 2.0 - residual[np.logical_not(case_1_u)] = t2 - np.sqrt((1.0 - u[np.logical_not(case_1_u)]) * (t2 * t2 - t1 * t1)) + residual[np.logical_not(case_1_u)] = t2 - np.sqrt( + (1.0 - u[np.logical_not(case_1_u)]) * (t2 * t2 - t1 * t1) + ) else: residual = u * mean return residual @@ -65,7 +71,9 @@ def initial_speed(rng, speed_mean, speed_delta, shape=(1,)): return pow(v1, u) / pow(v0, u - 1) -def init_random_waypoint(rng, nr_nodes, dimensions, speed_low, speed_high, pause_low, pause_high): +def init_random_waypoint( + rng, nr_nodes, dimensions, speed_low, speed_high, pause_low, pause_high +): ndim = len(dimensions) positions = np.empty((nr_nodes, ndim)) waypoints = np.empty((nr_nodes, ndim)) @@ -76,11 +84,17 @@ def init_random_waypoint(rng, nr_nodes, dimensions, speed_low, speed_high, pause speed_high = float(speed_high) moving = np.ones(nr_nodes) - speed_mean, speed_delta = (speed_low + speed_high) / 2.0, (speed_high - speed_low) / 2.0 - pause_mean, pause_delta = (pause_low + pause_high) / 2.0, (pause_high - pause_low) / 2.0 + speed_mean, speed_delta = (speed_low + speed_high) / 2.0, ( + speed_high - speed_low + ) / 2.0 + pause_mean, pause_delta = (pause_low + pause_high) / 2.0, ( + pause_high - pause_low + ) / 2.0 # steady-state pause probability for Random Waypoint - q0 = pause_probability_init(pause_low, pause_high, speed_low, speed_high, dimensions) + q0 = pause_probability_init( + pause_low, pause_high, speed_low, speed_high, dimensions + ) for i in range(nr_nodes): while True: @@ -109,7 +123,9 @@ def init_random_waypoint(rng, nr_nodes, dimensions, speed_low, speed_high, pause # steady-state speed and pause time paused_bool = moving == 0.0 paused_idx = np.where(paused_bool)[0] - pause_time[paused_idx] = residual_time(rng, pause_mean, pause_delta, paused_idx.shape) + pause_time[paused_idx] = residual_time( + rng, pause_mean, pause_delta, paused_idx.shape + ) speed[paused_idx] = 0.0 moving_bool = np.logical_not(paused_bool) @@ -219,7 +235,9 @@ def __iter__(self): velocity[arrived] = U(self.rng, MIN_V, MAX_V, arrived) new_direction = waypoints[arrived] - positions[arrived] - direction[arrived] = new_direction / np.linalg.norm(new_direction, axis=1)[:, np.newaxis] + direction[arrived] = ( + new_direction / np.linalg.norm(new_direction, axis=1)[:, np.newaxis] + ) self.velocity = velocity self.wt = wt @@ -652,7 +670,10 @@ def __init__( FL_MIN = FL_MAX / 10.0 def FL_DISTR(SAMPLES): - return rng.random(len(SAMPLES)) * (FL_MAX[SAMPLES] - FL_MIN[SAMPLES]) + FL_MIN[SAMPLES] + return ( + rng.random(len(SAMPLES)) * (FL_MAX[SAMPLES] - FL_MIN[SAMPLES]) + + FL_MIN[SAMPLES] + ) def WT_DISTR(SAMPLES): return P(rng, WT_EXP, 1.0, WT_MAX, SAMPLES) @@ -747,7 +768,9 @@ def gauss_markov( old_num_users = num_users num_users = int(num_users / len(anchor_loc)) * len(anchor_loc) if old_num_users != num_users: - logging.info("len(anchor_loc) must evenly divide num_users.....terminating....") + logging.info( + "len(anchor_loc) must evenly divide num_users.....terminating...." + ) return num_users_per_anchor = int(num_users / len(anchor_loc)) @@ -790,9 +813,17 @@ def gauss_markov( angle_mean[b] = -angle_mean[b] # calculate new speed and direction based on the model - velocity = alpha * velocity + alpha2 * velocity_mean + alpha3 * rng.normal(0.0, 1.0, num_users) + velocity = ( + alpha * velocity + + alpha2 * velocity_mean + + alpha3 * rng.normal(0.0, 1.0, num_users) + ) - theta = alpha * theta + alpha2 * angle_mean + alpha3 * rng.normal(0.0, 1.0, num_users) + theta = ( + alpha * theta + + alpha2 * angle_mean + + alpha3 * rng.normal(0.0, 1.0, num_users) + ) yield np.dstack((x, y))[0] @@ -814,7 +845,11 @@ def non_homogeneous_drop( user_loc = [] for anchor_it in range(num_anchors): anchor_mean = anchor_loc[anchor_it, :] - user_loc.append(rng.multivariate_normal(mean=anchor_mean, cov=cov_around_anchor, size=num_users_per_anchor)) + user_loc.append( + rng.multivariate_normal( + mean=anchor_mean, cov=cov_around_anchor, size=num_users_per_anchor + ) + ) user_loc = np.concatenate(user_loc, axis=0) diff --git a/radp/digital_twin/mobility/param_regression.py b/radp/digital_twin/mobility/param_regression.py index 72c8f03..41ca9d3 100644 --- a/radp/digital_twin/mobility/param_regression.py +++ b/radp/digital_twin/mobility/param_regression.py @@ -148,20 +148,19 @@ def preprocess_ue_data(df: pd.DataFrame) -> pd.DataFrame: return df - -def get_predicted_alpha(data: pd.DataFrame, alpha0: float,seed: int) -> float: +def get_predicted_alpha(data: pd.DataFrame, alpha0: float, seed: int) -> float: """ Get the predicted alpha value for the given UE (User Equipment) data. - This function serves as the main API entry point for predicting the alpha parameter, - which represents a key aspect of mobility behavior derived from UE tracks. + This function serves as the main API entry point for predicting the alpha parameter, + which represents a key aspect of mobility behavior derived from UE tracks. - Alpha is predicted based on the movement patterns observed in the UE tracks, + Alpha is predicted based on the movement patterns observed in the UE tracks, which are represented by the following columns in the input DataFrame: Parameters: - data (DataFrame): The input data containing UE tracks, structured as follows: - + +------------+------------+-----------+------+ | mock_ue_id | lon | lat | tick | +============+============+===========+======+ @@ -172,13 +171,12 @@ def get_predicted_alpha(data: pd.DataFrame, alpha0: float,seed: int) -> float: | 1 | 102.362725 | 33.916477 | 1 | | 2 | 102.080675 | 33.832793 | 1 | +------------+------------+-----------+------+ - -""" + """ # Extract the data after preprocessing velocity = preprocess_ue_data(data) # Initialize and unpack all outputs from the initialize function - t_array, t_next_array, velocity_mean, variance, rng = _initialize(velocity,seed) + t_array, t_next_array, velocity_mean, variance, rng = _initialize(velocity, seed) # Optimize alpha using the unpacked values popt, pcov = optimize_alpha( @@ -187,3 +185,4 @@ def get_predicted_alpha(data: pd.DataFrame, alpha0: float,seed: int) -> float: # Return the optimized alpha value return popt[0] + diff --git a/radp/digital_twin/mobility/tests/test_ue_tracks_generation_helper.py b/radp/digital_twin/mobility/tests/test_ue_tracks_generation_helper.py index 82d9276..57b054c 100644 --- a/radp/digital_twin/mobility/tests/test_ue_tracks_generation_helper.py +++ b/radp/digital_twin/mobility/tests/test_ue_tracks_generation_helper.py @@ -31,7 +31,6 @@ class TestUETracksGenerationHelper(unittest.TestCase): """ def setUp(self): - self.job_data = { constants.SIMULATION_ID: "1234", constants.UE_TRACKS_GENERATION: { diff --git a/radp/digital_twin/mobility/tests/test_ue_tracks_params.py b/radp/digital_twin/mobility/tests/test_ue_tracks_params.py index e69499a..1afc9f9 100644 --- a/radp/digital_twin/mobility/tests/test_ue_tracks_params.py +++ b/radp/digital_twin/mobility/tests/test_ue_tracks_params.py @@ -142,4 +142,4 @@ def test_initialization_and_extraction( self.assertEqual( params.mobility_class_velocity_variances[MobilityClass.cyclist], 1 ) - self.assertEqual(params.mobility_class_velocity_variances[MobilityClass.car], 1) \ No newline at end of file + self.assertEqual(params.mobility_class_velocity_variances[MobilityClass.car], 1) diff --git a/radp/digital_twin/mobility/ue_tracks.py b/radp/digital_twin/mobility/ue_tracks.py index db8d88e..2473291 100644 --- a/radp/digital_twin/mobility/ue_tracks.py +++ b/radp/digital_twin/mobility/ue_tracks.py @@ -98,7 +98,10 @@ def __init__( self.mobility_class_velocity_variances = mobility_class_velocity_variances self.sampled_users_per_mobility_class = self.rng.choice( - [mobility_class.value for mobility_class in list(self.mobility_class_distribution.keys())], + [ + mobility_class.value + for mobility_class in list(self.mobility_class_distribution.keys()) + ], size=(self.num_UEs), replace=True, p=list(self.mobility_class_distribution.values()), @@ -111,9 +114,17 @@ def __init__( # mapping the count of users and the velocity ranges # across for different mobility classes for k in self.mobility_class_distribution.keys(): - self.num_users_per_mobility_class[k] = np.count_nonzero(self.sampled_users_per_mobility_class == k.value) - low = self.mobility_class_velocities[k] - self.mobility_class_velocity_variances[k] - high = self.mobility_class_velocities[k] + self.mobility_class_velocity_variances[k] + self.num_users_per_mobility_class[k] = np.count_nonzero( + self.sampled_users_per_mobility_class == k.value + ) + low = ( + self.mobility_class_velocities[k] + - self.mobility_class_velocity_variances[k] + ) + high = ( + self.mobility_class_velocities[k] + + self.mobility_class_velocity_variances[k] + ) self.velocity_range[k] = [low, high] # mapping the gauss_markov models to their respective mobility classes diff --git a/radp/digital_twin/rf/bayesian/bayesian_engine.py b/radp/digital_twin/rf/bayesian/bayesian_engine.py index 8c5809b..9d93b60 100644 --- a/radp/digital_twin/rf/bayesian/bayesian_engine.py +++ b/radp/digital_twin/rf/bayesian/bayesian_engine.py @@ -116,7 +116,9 @@ def __init__( train_X, train_Y = self._create_training_tensors(data_in) # initialize likelihood and model - likelihood = gpytorch.likelihoods.GaussianLikelihood(batch_shape=torch.Size([self.num_cells])) + likelihood = gpytorch.likelihoods.GaussianLikelihood( + batch_shape=torch.Size([self.num_cells]) + ) self.model = ExactGPModel(train_X, train_Y, likelihood) @@ -127,20 +129,30 @@ def _create_training_tensors( n_train = data_in[0].shape[0] # Get train_X and train_Y, create training tensors - train_X = torch.zeros([self.num_cells, n_train, self.num_features], dtype=torch.float32) + train_X = torch.zeros( + [self.num_cells, n_train, self.num_features], dtype=torch.float32 + ) train_Y = torch.zeros([self.num_cells, n_train], dtype=torch.float32) for m in range(self.num_cells): if self.norm_method == NormMethod.MINMAX: - train_x_cell = (data_in[m][self.x_columns] - self.xmin[m]) / (self.xmax[m] - self.xmin[m]) + train_x_cell = (data_in[m][self.x_columns] - self.xmin[m]) / ( + self.xmax[m] - self.xmin[m] + ) elif self.norm_method == NormMethod.ZSCORE: - train_x_cell = (data_in[m][self.x_columns] - self.xmeans[m]) / self.xstds[m] + train_x_cell = ( + data_in[m][self.x_columns] - self.xmeans[m] + ) / self.xstds[m] - train_X_cell = torch.tensor(train_x_cell.iloc[:, :].values, dtype=torch.float32) + train_X_cell = torch.tensor( + train_x_cell.iloc[:, :].values, dtype=torch.float32 + ) train_y_cell = (data_in[m][self.y_columns] - self.ymeans[m]) / self.ystds[m] - train_Y_cell = torch.tensor(train_y_cell.iloc[:, :].values, dtype=torch.float32) + train_Y_cell = torch.tensor( + train_y_cell.iloc[:, :].values, dtype=torch.float32 + ) train_X[m] = train_X_cell.reshape(shape=(1, -1, self.num_features)) train_Y[m] = torch.transpose(train_Y_cell, 0, 1) @@ -148,7 +160,9 @@ def _create_training_tensors( return train_X, train_Y @staticmethod - def preprocess_ue_training_data(ue_training_data_df: pd.DataFrame, topology_df: pd.DataFrame) -> Dict: + def preprocess_ue_training_data( + ue_training_data_df: pd.DataFrame, topology_df: pd.DataFrame + ) -> Dict: """Preprocess UE data before training ue_training_data_df - dataframe containing location data as well as config @@ -159,7 +173,9 @@ def preprocess_ue_training_data(ue_training_data_df: pd.DataFrame, topology_df: # feature engineering -- add relative bearing and distance for i in ue_training_data_df.index: - cell_topology = topology_df[topology_df.cell_id == ue_training_data_df.at[i, "cell_id"]] + cell_topology = topology_df[ + topology_df.cell_id == ue_training_data_df.at[i, "cell_id"] + ] # change lon/lat to loc_x/loc_y in ue data ue_training_data_df.at[i, "loc_x"] = ue_training_data_df.at[i, "lon"] @@ -168,8 +184,12 @@ def preprocess_ue_training_data(ue_training_data_df: pd.DataFrame, topology_df: # add the topology columns to training data ue_training_data_df.at[i, "cell_lat"] = cell_topology.cell_lat.values[0] ue_training_data_df.at[i, "cell_lon"] = cell_topology.cell_lon.values[0] - ue_training_data_df.at[i, "cell_az_deg"] = cell_topology.cell_az_deg.values[0] - ue_training_data_df.at[i, "cell_carrier_freq_mhz"] = cell_topology.cell_carrier_freq_mhz.values[0] + ue_training_data_df.at[i, "cell_az_deg"] = cell_topology.cell_az_deg.values[ + 0 + ] + ue_training_data_df.at[ + i, "cell_carrier_freq_mhz" + ] = cell_topology.cell_carrier_freq_mhz.values[0] # engineer and add the log distance and relative bearing features ue_training_data_df.at[i, "log_distance"] = np.log( @@ -225,7 +245,9 @@ def preprocess_ue_prediction_data( for i in ue_data_df.index: # pull the config and topology for this cell cell_config = config_df[config_df.cell_id == ue_data_df.at[i, "cell_id"]] - cell_topology = topology_df[topology_df.cell_id == ue_data_df.at[i, "cell_id"]] + cell_topology = topology_df[ + topology_df.cell_id == ue_data_df.at[i, "cell_id"] + ] # change lon/lat to loc_x/loc_y in ue data ue_data_df.at[i, "loc_x"] = ue_data_df.at[i, "lon"] @@ -235,7 +257,9 @@ def preprocess_ue_prediction_data( ue_data_df.at[i, "cell_lat"] = cell_topology.cell_lat.values[0] ue_data_df.at[i, "cell_lon"] = cell_topology.cell_lon.values[0] ue_data_df.at[i, "cell_az_deg"] = cell_topology.cell_az_deg.values[0] - ue_data_df.at[i, "cell_carrier_freq_mhz"] = cell_topology.cell_carrier_freq_mhz.values[0] + ue_data_df.at[ + i, "cell_carrier_freq_mhz" + ] = cell_topology.cell_carrier_freq_mhz.values[0] # add the config columns to ue data ue_data_df.at[i, "cell_el_deg"] = cell_config.cell_el_deg.values[0] @@ -281,7 +305,9 @@ def load_model_map_from_pickle( model_map: Dict[str, BayesianDigitalTwin] = pickle.load(pickle_file) return model_map except Exception as e: - logger.exception(f"Exception occurred while loading digital twin model from file: {model_file_path}") + logger.exception( + f"Exception occurred while loading digital twin model from file: {model_file_path}" + ) raise e @staticmethod @@ -295,7 +321,9 @@ def save_model_map_to_pickle( pickle.dump(model_map, pickle_file) logger.info(f"Successfully saved model to file: {model_file_path}") except Exception as e: - logger.exception(f"Exception occurred writing digital twin model to file: {model_file_path}") + logger.exception( + f"Exception occurred writing digital twin model to file: {model_file_path}" + ) raise e @staticmethod @@ -331,9 +359,15 @@ def split_training_and_test_data( """ n_training_group = np.max([int(alpha * 0.01 * n_sim), 1]) n_test_group = n_sim - n_training_group - logger.info(f"Splitting data into {n_training_group} training and {n_test_group} test groups...") - training_data = data_in[data_in[constants.SIM_IDX] > n_test_group].reset_index(drop=True) - test_data = data_in[data_in[constants.SIM_IDX] <= n_test_group].reset_index(drop=True) + logger.info( + f"Splitting data into {n_training_group} training and {n_test_group} test groups..." + ) + training_data = data_in[data_in[constants.SIM_IDX] > n_test_group].reset_index( + drop=True + ) + test_data = data_in[data_in[constants.SIM_IDX] <= n_test_group].reset_index( + drop=True + ) stats = data_in.describe(include="all") return training_data, test_data, stats, n_training_group @@ -370,7 +404,9 @@ def create_prediction_frames( lat, lon, ) - for lat, lon in zip(prediction_frame_template.loc_y, prediction_frame_template.loc_x) + for lat, lon in zip( + prediction_frame_template.loc_y, prediction_frame_template.loc_x + ) ] prediction_df[constants.RELATIVE_BEARING] = [ @@ -381,7 +417,9 @@ def create_prediction_frames( lat, lon, ) - for lat, lon in zip(prediction_frame_template.loc_y, prediction_frame_template.loc_x) + for lat, lon in zip( + prediction_frame_template.loc_y, prediction_frame_template.loc_x + ) ] prediction_df[constants.ANTENNA_GAIN] = GISTools.get_antenna_gain( @@ -413,7 +451,9 @@ def train_distributed_gpmodel( if load_model: # Check that model path and name are both provided if not model_path or not model_name: - raise RuntimeError("Exception loading model: model_path and model_name must be provided") + raise RuntimeError( + "Exception loading model: model_path and model_name must be provided" + ) logger.info("Now loading GP model (this should be quick...)") state_dict = torch.load(model_path + model_name) self.model.load_state_dict(state_dict) @@ -423,7 +463,9 @@ def train_distributed_gpmodel( else: self.model.train() # "Loss" for GPs - the marginal log likelihood - mll = gpytorch.mlls.ExactMarginalLogLikelihood(self.model.likelihood, self.model) + mll = gpytorch.mlls.ExactMarginalLogLikelihood( + self.model.likelihood, self.model + ) optimizer = torch.optim.Adam(self.model.parameters(), lr=lr) train_X, train_Y = ( @@ -449,7 +491,10 @@ def train_distributed_gpmodel( loss_vs_iter[i] = this_loss delta = this_loss - last_loss last_loss = this_loss - logger.info("Iter %d/%d - Loss: %.3f (delta=%.6f)" % (i + 1, maxiter, this_loss, delta)) + logger.info( + "Iter %d/%d - Loss: %.3f (delta=%.6f)" + % (i + 1, maxiter, this_loss, delta) + ) if abs(delta) < stopping_threshold: logger.info("Stopping criteria met...exiting.") break @@ -458,7 +503,9 @@ def train_distributed_gpmodel( if save_model: # Check that model path and name are both provided if not model_path or not model_name: - raise RuntimeError("Exception saving model: model_path and model_name must be provided") + raise RuntimeError( + "Exception saving model: model_path and model_name must be provided" + ) if not os.path.exists(model_path): os.makedirs(model_path) torch.save(self.model.state_dict(), model_path + model_name) @@ -516,15 +563,23 @@ def predict_distributed_gpmodel( num_locations = prediction_dfs[0].shape[0] pred_means = torch.zeros([num_locations, self.num_cells], dtype=torch.float32) pred_stds = torch.zeros([num_locations, self.num_cells], dtype=torch.float32) - predict_X = torch.zeros([self.num_cells, num_locations, self.num_features], dtype=torch.float32) + predict_X = torch.zeros( + [self.num_cells, num_locations, self.num_features], dtype=torch.float32 + ) for m in range(self.num_cells): if self.norm_method == NormMethod.MINMAX: - predict_x_cell = (prediction_dfs[m][self.x_columns] - self.xmin[m]) / (self.xmax[m] - self.xmin[m]) + predict_x_cell = (prediction_dfs[m][self.x_columns] - self.xmin[m]) / ( + self.xmax[m] - self.xmin[m] + ) elif self.norm_method == NormMethod.ZSCORE: - predict_x_cell = (prediction_dfs[m][self.x_columns] - self.xmeans[m]) / self.xstds[m] + predict_x_cell = ( + prediction_dfs[m][self.x_columns] - self.xmeans[m] + ) / self.xstds[m] - predict_X_cell = torch.tensor(predict_x_cell.iloc[:, :].values, dtype=torch.float32) + predict_X_cell = torch.tensor( + predict_x_cell.iloc[:, :].values, dtype=torch.float32 + ) predict_X[m] = predict_X_cell.reshape(shape=(1, -1, self.num_features)) if self.is_cuda: diff --git a/radp/digital_twin/rf/bayesian/tests/test_bayesian_engine.py b/radp/digital_twin/rf/bayesian/tests/test_bayesian_engine.py index 860ca15..b31afbd 100644 --- a/radp/digital_twin/rf/bayesian/tests/test_bayesian_engine.py +++ b/radp/digital_twin/rf/bayesian/tests/test_bayesian_engine.py @@ -147,7 +147,9 @@ def augment_ue_data(ue_data_df: pd.DataFrame, site_configs_df: pd.DataFrame): """ for i in ue_data_df.index: - site_config = site_configs_df[site_configs_df.cell_id == ue_data_df.at[i, "cell_id"]] + site_config = site_configs_df[ + site_configs_df.cell_id == ue_data_df.at[i, "cell_id"] + ] ue_data_df.at[i, "cell_id"] = site_config.cell_id.values[0] ue_data_df.at[i, "loc_x"] = ue_data_df.at[i, "lon"] ue_data_df.at[i, "loc_y"] = ue_data_df.at[i, "lat"] @@ -155,7 +157,9 @@ def augment_ue_data(ue_data_df: pd.DataFrame, site_configs_df: pd.DataFrame): ue_data_df.at[i, "cell_lon"] = site_config.cell_lon.values[0] ue_data_df.at[i, "cell_az_deg"] = site_config.cell_az_deg.values[0] ue_data_df.at[i, "cell_el_deg"] = site_config.cell_el_deg.values[0] - ue_data_df.at[i, "cell_carrier_freq_mhz"] = site_config.cell_carrier_freq_mhz.values[0] + ue_data_df.at[ + i, "cell_carrier_freq_mhz" + ] = site_config.cell_carrier_freq_mhz.values[0] ue_data_df.at[i, "log_distance"] = GISTools.get_log_distance( ue_data_df.at[i, "cell_lat"], @@ -210,20 +214,22 @@ def setUpClass(cls): logging.info(ue_data_df) # split into training/test - cls.cell_id_training_data_map, cls.cell_id_test_data_map = split_training_and_test_data(ue_data_df, 0.2) + ( + cls.cell_id_training_data_map, + cls.cell_id_test_data_map, + ) = split_training_and_test_data(ue_data_df, 0.2) # Generate test data expected_loss_vs_iter, expected_mae and expected_mape with seed_everything(1) cls.expected_loss_vs_iter = {} cls.expected_mae = {} cls.expected_mape = {} - def produce_results_rf_bayesian_digital_twin(self): # train bayesian_digital_twin_map = {} x_max, x_min = get_x_max_and_x_min() - + for cell_id, training_data in self.cell_id_training_data_map.items(): bayesian_digital_twin_map[cell_id] = BayesianDigitalTwin( data_in=[training_data], @@ -250,19 +256,24 @@ def produce_results_rf_bayesian_digital_twin(self): # predict/test for cell_id, testing_data in self.cell_id_test_data_map.items(): - (pred_means, _) = bayesian_digital_twin_map[cell_id].predict_distributed_gpmodel( - prediction_dfs=[testing_data] - ) + (pred_means, _) = bayesian_digital_twin_map[ + cell_id + ].predict_distributed_gpmodel(prediction_dfs=[testing_data]) MAE = abs(testing_data.avg_rsrp - pred_means[0]).mean() # mean absolute percentage error - MAPE = 100 * abs((testing_data.avg_rsrp - pred_means[0]) / testing_data.avg_rsrp).mean() + MAPE = ( + 100 + * abs( + (testing_data.avg_rsrp - pred_means[0]) / testing_data.avg_rsrp + ).mean() + ) logging.info( f"cell_id = {cell_id}, MAE = {MAE:0.5f} dB, MAPE = {MAPE:0.5f} %," "# test points = {len(testing_data.avg_rsrp)}" ) self.expected_mae[cell_id] = MAE self.expected_mape[cell_id] = MAPE - + def test_reproducibility_of_results_for_bayesian_digital_twin(self): # produce results and re_run to verify reproducibility of result using seed(1) self.produce_results_rf_bayesian_digital_twin() @@ -270,7 +281,7 @@ def test_reproducibility_of_results_for_bayesian_digital_twin(self): bayesian_digital_twin_map = {} x_max, x_min = get_x_max_and_x_min() - + for cell_id, training_data in self.cell_id_training_data_map.items(): bayesian_digital_twin_map[cell_id] = BayesianDigitalTwin( data_in=[training_data], @@ -297,16 +308,20 @@ def test_reproducibility_of_results_for_bayesian_digital_twin(self): # predict/test for cell_id, testing_data in self.cell_id_test_data_map.items(): - (pred_means, _) = bayesian_digital_twin_map[cell_id].predict_distributed_gpmodel( - prediction_dfs=[testing_data] - ) + (pred_means, _) = bayesian_digital_twin_map[ + cell_id + ].predict_distributed_gpmodel(prediction_dfs=[testing_data]) MAE = abs(testing_data.avg_rsrp - pred_means[0]).mean() # mean absolute percentage error - MAPE = 100 * abs((testing_data.avg_rsrp - pred_means[0]) / testing_data.avg_rsrp).mean() + MAPE = ( + 100 + * abs( + (testing_data.avg_rsrp - pred_means[0]) / testing_data.avg_rsrp + ).mean() + ) logging.info( f"cell_id = {cell_id}, MAE = {MAE:0.5f} dB, MAPE = {MAPE:0.5f} %," "# test points = {len(testing_data.avg_rsrp)}" ) self.assertEqual(self.expected_mae[cell_id], MAE) self.assertEqual(self.expected_mape[cell_id], MAPE) - diff --git a/radp/digital_twin/utils/cell_selection.py b/radp/digital_twin/utils/cell_selection.py index 7fde4a9..14ca5bb 100644 --- a/radp/digital_twin/utils/cell_selection.py +++ b/radp/digital_twin/utils/cell_selection.py @@ -48,12 +48,14 @@ def perform_attachment( """ # initiate a dictionary to store power-by-layer dictionaries on a per-pixel basis - rx_powers_by_layer_by_loc: Dict[Tuple[float, float], Dict[float, List[Tuple[Any, float]]]] = defaultdict( - lambda: defaultdict(list) - ) + rx_powers_by_layer_by_loc: Dict[ + Tuple[float, float], Dict[float, List[Tuple[Any, float]]] + ] = defaultdict(lambda: defaultdict(list)) # pull per-cell frequencies for faster lookup - cell_id_to_freq = {row.cell_id: row.cell_carrier_freq_mhz for _, row in topology.iterrows()} + cell_id_to_freq = { + row.cell_id: row.cell_carrier_freq_mhz for _, row in topology.iterrows() + } # iterate over ue_prediction_data, to # build rx_powers_by_layer_by_loc map @@ -68,19 +70,25 @@ def perform_attachment( raise Exception("loc_x or loc_y cannot be found in the dataset") # add (cell_id, rxpower) tuple on a per-row, per-freq basis - rx_powers_by_layer_by_loc[(loc_x, loc_y)][cell_carrier_freq_mhz].append((row.cell_id, row.rxpower_dbm)) + rx_powers_by_layer_by_loc[(loc_x, loc_y)][cell_carrier_freq_mhz].append( + (row.cell_id, row.rxpower_dbm) + ) # perform cell selection per location rf_dataframe_dict = defaultdict(list) for loc, rx_powers_by_layer in rx_powers_by_layer_by_loc.items(): # compute strongest server, interference and SINR - rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer(rx_powers_by_layer) + rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer( + rx_powers_by_layer + ) # pull sinr_db, cell_id and rsrp_dbm based on highest SINR max_sinr_db_item = max(sinr_db_by_layer.items(), key=lambda k: k[1][1]) max_sinr_db_cell_id, max_sinr_db = max_sinr_db_item[1] - rsrp_dbm = next(v[1] for v in rsrp_dbm_by_layer.values() if v[0] == max_sinr_db_cell_id) + rsrp_dbm = next( + v[1] for v in rsrp_dbm_by_layer.values() if v[0] == max_sinr_db_cell_id + ) # update rf_dataframe output rf_dataframe_dict[constants.LOC_X].append(loc[0]) diff --git a/radp/digital_twin/utils/gis_tools.py b/radp/digital_twin/utils/gis_tools.py index 9524d56..8c21f21 100644 --- a/radp/digital_twin/utils/gis_tools.py +++ b/radp/digital_twin/utils/gis_tools.py @@ -52,7 +52,9 @@ class GISTools: def get_tile_side_length_meters(bing_tile_zoom: int) -> float: """Returns equatorial ground length (in meters) for the specified Bing Tile zoom level.""" - assert bing_tile_zoom <= 20, "Only Bing Tile Zoom Level 20 and coarser are supported!" + assert ( + bing_tile_zoom <= 20 + ), "Only Bing Tile Zoom Level 20 and coarser are supported!" return GISTools.bing_tile_zoom_to_ground_resolution_meters_dict[bing_tile_zoom] @staticmethod @@ -61,19 +63,27 @@ def get_tile_side_length_km(lat: float, zoom: int) -> float: Given a latitude and zoom level, return the side length of a Bing tile at that zoom level. """ - return float(math.cos(lat * math.pi / 180) * 2 * math.pi * GISTools.R / (2**zoom)) + return float( + math.cos(lat * math.pi / 180) * 2 * math.pi * GISTools.R / (2**zoom) + ) @staticmethod - def isclose(A: Tuple[float, float], B: Tuple[float, float], abs_tol: float = 0.0002) -> bool: + def isclose( + A: Tuple[float, float], B: Tuple[float, float], abs_tol: float = 0.0002 + ) -> bool: try: - return math.isclose(A[0], B[0], abs_tol=abs_tol) and math.isclose(A[1], B[1], abs_tol=abs_tol) - except AttributeError: - return abs(A[0] - B[0]) <= max(1e-9 * max(abs(A[0]), abs(B[0])), abs_tol) and abs(A[1] - B[1]) <= max( - 1e-9 * max(abs(A[1]), abs(B[1])), abs_tol + return math.isclose(A[0], B[0], abs_tol=abs_tol) and math.isclose( + A[1], B[1], abs_tol=abs_tol ) + except AttributeError: + return abs(A[0] - B[0]) <= max( + 1e-9 * max(abs(A[0]), abs(B[0])), abs_tol + ) and abs(A[1] - B[1]) <= max(1e-9 * max(abs(A[1]), abs(B[1])), abs_tol) @staticmethod - def dist(l1: Tuple[float, float], l2: Tuple[float, float], abs_tol: float = 0.0002) -> float: + def dist( + l1: Tuple[float, float], l2: Tuple[float, float], abs_tol: float = 0.0002 + ) -> float: """Returns distance (in kms) between two points on the earth. Utlizes haversine formula (https://en.wikipedia.org/wiki/Haversine_formula) @@ -117,12 +127,16 @@ def get_bearing(l1: Tuple[float, float], l2: Tuple[float, float]) -> float: [phi1, lam1] = [math.radians(l1[0]), math.radians(l1[1])] [phi2, lam2] = [math.radians(l2[0]), math.radians(l2[1])] y = math.sin(lam2 - lam1) * math.cos(phi2) - x = math.cos(phi1) * math.sin(phi2) - math.sin(phi1) * math.cos(phi2) * math.cos(lam2 - lam1) + x = math.cos(phi1) * math.sin(phi2) - math.sin(phi1) * math.cos( + phi2 + ) * math.cos(lam2 - lam1) return math.degrees(math.atan2(y, x)) @staticmethod def get_destination( - origin: Union[List[int], List[float], Tuple[Union[int, float], Union[int, float]]], + origin: Union[ + List[int], List[float], Tuple[Union[int, float], Union[int, float]] + ], brng: float, d: float, ) -> Tuple[float, float]: @@ -136,7 +150,10 @@ def get_destination( R = GISTools.R brng_r = math.radians(brng) [phi1, lam1] = [math.radians(origin[0]), math.radians(origin[1])] - phi2 = math.asin(math.sin(phi1) * math.cos(d / R) + math.cos(phi1) * math.sin(d / R) * math.cos(brng_r)) + phi2 = math.asin( + math.sin(phi1) * math.cos(d / R) + + math.cos(phi1) * math.sin(d / R) * math.cos(brng_r) + ) lam2 = lam1 + math.atan2( math.sin(brng_r) * math.sin(d / R) * math.cos(phi1), math.cos(d / R) - (math.sin(phi1) * math.sin(phi2)), @@ -195,7 +212,9 @@ def random_location( return (random_lon, random_lat) @staticmethod - def snap_align_lower_left(pt: Tuple[float, float], tile_discretization_resolution: int) -> Tuple[float, float]: + def snap_align_lower_left( + pt: Tuple[float, float], tile_discretization_resolution: int + ) -> Tuple[float, float]: lat = int(pt[0]) lon = int(pt[1]) inc = float(1 / float(tile_discretization_resolution - 1)) @@ -282,7 +301,9 @@ def mk_grid_params( """Create aligned and adjusted grid params""" inc = 1.0 / float(tile_discretization_resolution - 1) SW = GISTools.snap_align_lower_left(SW, tile_discretization_resolution) - NE = GISTools.snap_align_lower_left((NE[0] + inc, NE[1] + inc), tile_discretization_resolution) + NE = GISTools.snap_align_lower_left( + (NE[0] + inc, NE[1] + inc), tile_discretization_resolution + ) inc *= coarse_factor num_rows = int(math.ceil((NE[0] - SW[0]) / inc)) + 1 num_cols = int(math.ceil((NE[1] - SW[1]) / inc)) + 1 @@ -298,7 +319,9 @@ def get_bounding_box( num_cols: int, tile_discretization_resolution: int, ) -> Tuple[Tuple[float, float], Tuple[float, float]]: - aligned = GISTools.snap_align_lower_left((lat, lon), tile_discretization_resolution) + aligned = GISTools.snap_align_lower_left( + (lat, lon), tile_discretization_resolution + ) idx = GISTools.get_grid_idx(aligned, SW, tile_discretization_resolution) box_radius = math.floor(radius / 30.0) if math.isclose(box_radius, 0): @@ -335,7 +358,12 @@ def coord_str(coord: float) -> str: mantissa_decimal = mantissa.split(".")[1] trailing_zeros_to_add = "0" * (16 - len(mantissa_decimal)) return ( - mantissa.split(".")[0] + "." + mantissa_decimal + trailing_zeros_to_add + "e" + str_coord_parts[1] + mantissa.split(".")[0] + + "." + + mantissa_decimal + + trailing_zeros_to_add + + "e" + + str_coord_parts[1] ) else: # normal case return (precision_str_format % coord).rstrip("0").rstrip(".") @@ -345,7 +373,9 @@ def coord_str(coord: float) -> str: return "POINT (" + lon_str + " " + lat_str + ")" @staticmethod - def get_bbox_km_around_point(lat: float, lon: float, d: float) -> Tuple[Tuple[float, float], Tuple[float, float]]: + def get_bbox_km_around_point( + lat: float, lon: float, d: float + ) -> Tuple[Tuple[float, float], Tuple[float, float]]: """ Given a latlon point and a distance d (in km), return the SW and NE corners of a box whose side lengths are 2d with lat, lon as the center. @@ -385,7 +415,9 @@ def extend_bbox( return (minlat - s_deg, minlon - w_deg), (maxlat + n_deg, maxlon + e_deg) @staticmethod - def lon_lat_to_bing_tile(longitude: float, latitude: float, level: int = 18) -> Tuple[int, int]: + def lon_lat_to_bing_tile( + longitude: float, latitude: float, level: int = 18 + ) -> Tuple[int, int]: """Convert the given pair of longitude and latitude to Bing tile, at specified resolution. Technical Outline:- https://docs.microsoft.com/en-us/bingmaps/articles/bing-maps-tile-system @@ -439,7 +471,9 @@ def make_tile( tuple of tile_x(loc_x="longitude") and tile_y(loc_y="latitude") """ - tile_x, tile_y = GISTools.lon_lat_to_bing_tile(lon_lat_tuple[0], lon_lat_tuple[1], level=level) + tile_x, tile_y = GISTools.lon_lat_to_bing_tile( + lon_lat_tuple[0], lon_lat_tuple[1], level=level + ) return (tile_x, tile_y) @staticmethod @@ -528,7 +562,9 @@ def get_antenna_gain(hTx, hRx, log_distance, tilt_deg, theta_3db=10): tilt_deg: downtilt theta_3db=3dB bandwidth of Tx antenna """ - relative_tilt = np.degrees(np.arctan((hTx - hRx) / np.exp(log_distance))) - tilt_deg + relative_tilt = ( + np.degrees(np.arctan((hTx - hRx) / np.exp(log_distance))) - tilt_deg + ) G_db = -12 * np.power(relative_tilt / theta_3db, 2) return G_db diff --git a/radp/digital_twin/utils/tests/test_cell_selection.py b/radp/digital_twin/utils/tests/test_cell_selection.py index 0837160..98fb3b5 100644 --- a/radp/digital_twin/utils/tests/test_cell_selection.py +++ b/radp/digital_twin/utils/tests/test_cell_selection.py @@ -24,7 +24,10 @@ import pandas as pd from radp.digital_twin.utils import constants -from radp.digital_twin.utils.cell_selection import get_rsrp_dbm_sinr_db_by_layer, perform_attachment +from radp.digital_twin.utils.cell_selection import ( + get_rsrp_dbm_sinr_db_by_layer, + perform_attachment, +) class TestCellSelection(unittest.TestCase): @@ -33,8 +36,12 @@ def test_get_rsrp_dbm_sinr_db_by_layer(self): freq = 2100 # 1. 1 layer, 2 equal powered cells - rx_powers_by_layer: Dict[float, List[Tuple[str, float]]] = {freq: [("A", rx_dbm), ("B", rx_dbm)]} - rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer(rx_powers_by_layer) + rx_powers_by_layer: Dict[float, List[Tuple[str, float]]] = { + freq: [("A", rx_dbm), ("B", rx_dbm)] + } + rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer( + rx_powers_by_layer + ) self.assertEqual(len(rsrp_dbm_by_layer), 1) # 1 layer self.assertEqual(len(sinr_db_by_layer), 1) # 1 layer self.assertTrue(freq in rsrp_dbm_by_layer) # layer unchanged @@ -43,8 +50,12 @@ def test_get_rsrp_dbm_sinr_db_by_layer(self): self.assertAlmostEqual(sinr_db_by_layer[freq][1], 0) # SINR is very close to 0 # 2. 1 layer, one cell is twice the other in dbm scale - rx_powers_by_layer: Dict[float, List[Tuple[str, float]]] = {freq: [("A", rx_dbm), ("B", 2 * rx_dbm)]} - rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer(rx_powers_by_layer) + rx_powers_by_layer: Dict[float, List[Tuple[str, float]]] = { + freq: [("A", rx_dbm), ("B", 2 * rx_dbm)] + } + rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer( + rx_powers_by_layer + ) self.assertEqual(len(rsrp_dbm_by_layer), 1) # 1 layer self.assertEqual(len(sinr_db_by_layer), 1) # 1 layer self.assertTrue(freq in rsrp_dbm_by_layer) # layer unchanged @@ -52,14 +63,18 @@ def test_get_rsrp_dbm_sinr_db_by_layer(self): self.assertEqual(rsrp_dbm_by_layer[freq][0], "B") # bigger one wins self.assertEqual(rsrp_dbm_by_layer[freq][1], 2 * rx_dbm) # bigger one wins self.assertAlmostEqual(sinr_db_by_layer[freq][0], "B") # SINR winner is same - self.assertAlmostEqual(sinr_db_by_layer[freq][1], rx_dbm) # SINR is difference between bigger and smaller + self.assertAlmostEqual( + sinr_db_by_layer[freq][1], rx_dbm + ) # SINR is difference between bigger and smaller # 3. 2 layers, second cell 3x stronger for layer 1, first cell 3x stronger for layer 2 rx_powers_by_layer: Dict[float, List[Tuple[str, float]]] = { freq: [("A", rx_dbm), ("B", 3 * rx_dbm)], freq * 2: [("A2", 3 * rx_dbm), ("B2", rx_dbm)], } - rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer(rx_powers_by_layer) + rsrp_dbm_by_layer, sinr_db_by_layer = get_rsrp_dbm_sinr_db_by_layer( + rx_powers_by_layer + ) self.assertEqual(len(rsrp_dbm_by_layer), 2) # 2 layers self.assertEqual(len(sinr_db_by_layer), 2) # 2 layers # layers unchanged @@ -69,7 +84,9 @@ def test_get_rsrp_dbm_sinr_db_by_layer(self): ) self.assertEqual(rsrp_dbm_by_layer[freq][0], "B") # bigger one wins self.assertEqual(rsrp_dbm_by_layer[freq][1], 3 * rx_dbm) # bigger one wins - self.assertAlmostEqual(sinr_db_by_layer[freq][1], 2 * rx_dbm) # SINR is difference between bigger and smaller + self.assertAlmostEqual( + sinr_db_by_layer[freq][1], 2 * rx_dbm + ) # SINR is difference between bigger and smaller self.assertAlmostEqual(rsrp_dbm_by_layer[freq * 2][0], "A2") # bigger one wins self.assertEqual(rsrp_dbm_by_layer[freq * 2][1], 3 * rx_dbm) # bigger one wins self.assertAlmostEqual( diff --git a/radp/digital_twin/utils/tests/test_gis_tools.py b/radp/digital_twin/utils/tests/test_gis_tools.py index 8ac1f49..e2c22e8 100644 --- a/radp/digital_twin/utils/tests/test_gis_tools.py +++ b/radp/digital_twin/utils/tests/test_gis_tools.py @@ -119,7 +119,6 @@ def test_random_location(self) -> None: self.assertEqual(rand_lon_2_seed1, rand_lon_2_seed1_again) def test_bearing_utils(self) -> None: - # for get_bearing, the first parameter in the tuples corresponds to lat (y axis) self.assertEqual( @@ -226,7 +225,6 @@ def test_get_all_covering_tiles(self): self.assertTrue(tiles == expected) def test_converting_xy_points_into_lonlat_pairs(self): - s = [ np.array([85.8649796, 9.34373949]), np.array([69.74819822, 97.51281031]), diff --git a/radp/example_bayesian_engine_driver_script.py b/radp/example_bayesian_engine_driver_script.py index edc4e93..dfa2914 100755 --- a/radp/example_bayesian_engine_driver_script.py +++ b/radp/example_bayesian_engine_driver_script.py @@ -15,7 +15,11 @@ from radp.digital_twin import logger from radp.digital_twin.rf.bayesian.bayesian_engine import BayesianDigitalTwin -from radp.digital_twin.utils.constants import CELL_EL_DEG, LOG_DISTANCE, RELATIVE_BEARING +from radp.digital_twin.utils.constants import ( + CELL_EL_DEG, + LOG_DISTANCE, + RELATIVE_BEARING, +) from radp.digital_twin.utils.gis_tools import GISTools @@ -116,7 +120,9 @@ def augment_ue_data(ue_data_df: pd.DataFrame, site_configs_df: pd.DataFrame): """ for i in ue_data_df.index: - site_config = site_configs_df[site_configs_df.cell_id == ue_data_df.at[i, "cell_id"]] + site_config = site_configs_df[ + site_configs_df.cell_id == ue_data_df.at[i, "cell_id"] + ] ue_data_df.at[i, "cell_id"] = site_config.cell_id.values[0] ue_data_df.at[i, "loc_x"] = ue_data_df.at[i, "lon"] ue_data_df.at[i, "loc_y"] = ue_data_df.at[i, "lat"] @@ -124,7 +130,9 @@ def augment_ue_data(ue_data_df: pd.DataFrame, site_configs_df: pd.DataFrame): ue_data_df.at[i, "cell_lon"] = site_config.cell_lon.values[0] ue_data_df.at[i, "cell_az_deg"] = site_config.cell_az_deg.values[0] ue_data_df.at[i, "cell_el_deg"] = site_config.cell_el_deg.values[0] - ue_data_df.at[i, "cell_carrier_freq_mhz"] = site_config.cell_carrier_freq_mhz.values[0] + ue_data_df.at[ + i, "cell_carrier_freq_mhz" + ] = site_config.cell_carrier_freq_mhz.values[0] ue_data_df.at[i, "log_distance"] = GISTools.get_log_distance( ue_data_df.at[i, "cell_lat"], @@ -196,7 +204,9 @@ def get_x_max_and_x_min(): site_configs_df, ue_data_df = get_sample_site_config_and_ue_data() site_configs_df.reset_index(drop=True, inplace=True) - idx_cell_id_mapping = dict(zip(site_configs_df.index.values, site_configs_df.cell_id)) + idx_cell_id_mapping = dict( + zip(site_configs_df.index.values, site_configs_df.cell_id) + ) # feature engineering -- add relative bearing and distance augment_ue_data(ue_data_df, site_configs_df) @@ -204,7 +214,9 @@ def get_x_max_and_x_min(): logger.info(ue_data_df) # split into training/test - cell_id_training_data_map, cell_id_test_data_map = split_training_and_test_data(ue_data_df, 0.2) + cell_id_training_data_map, cell_id_test_data_map = split_training_and_test_data( + ue_data_df, 0.2 + ) # train bayesian_digital_twin_map = {} @@ -233,10 +245,17 @@ def get_x_max_and_x_min(): # predict/test for cell_id, testing_data in cell_id_test_data_map.items(): - (pred_means, _) = bayesian_digital_twin_map[cell_id].predict_distributed_gpmodel(prediction_dfs=[testing_data]) + (pred_means, _) = bayesian_digital_twin_map[ + cell_id + ].predict_distributed_gpmodel(prediction_dfs=[testing_data]) MAE = abs(testing_data.avg_rsrp - pred_means[0]).mean() # mean absolute percentage error - MAPE = 100 * abs((testing_data.avg_rsrp - pred_means[0]) / testing_data.avg_rsrp).mean() + MAPE = ( + 100 + * abs( + (testing_data.avg_rsrp - pred_means[0]) / testing_data.avg_rsrp + ).mean() + ) logger.info( f"cell_id = {cell_id}, MAE = {MAE:0.5f} dB, MAPE = {MAPE:0.5f} %, " f"# test points = {len(testing_data.avg_rsrp)}" diff --git a/radp/utility/kafka_utils.py b/radp/utility/kafka_utils.py index 7239302..a4a7cec 100644 --- a/radp/utility/kafka_utils.py +++ b/radp/utility/kafka_utils.py @@ -56,15 +56,23 @@ def safe_subscribe(consumer: kafka.Consumer, topics: List[str]): while not all([topic_name in topics_found for topic_name in topics]): current_attempt += 1 if current_attempt >= MAX_ATTEMPTS: - logger.exception(f"Timed out while attempting to subscribe consumer '{consumer}' to topics: {topics}") - raise Exception(f"Timed out while attempting to subscribe consumer '{consumer}' to topics: {topics}") + logger.exception( + f"Timed out while attempting to subscribe consumer '{consumer}' to topics: {topics}" + ) + raise Exception( + f"Timed out while attempting to subscribe consumer '{consumer}' to topics: {topics}" + ) time.sleep(SLEEP_INTERVAL) - topics_found = [topic_metadata for topic_metadata in consumer.list_topics().topics] + topics_found = [ + topic_metadata for topic_metadata in consumer.list_topics().topics + ] # all topics exist, subscribe to them try: consumer.subscribe(topics) except Exception as e: - logger.exception(f"Exception occurred while attempting to subscribe to topics: {e}") + logger.exception( + f"Exception occurred while attempting to subscribe to topics: {e}" + ) raise e diff --git a/radp/utility/pandas_utils.py b/radp/utility/pandas_utils.py index 85839d2..e542082 100644 --- a/radp/utility/pandas_utils.py +++ b/radp/utility/pandas_utils.py @@ -37,7 +37,9 @@ def cross_replicate(df_a: pd.DataFrame, df_b: pd.DataFrame) -> pd.DataFrame: """Cross replicate two pandas dataframes""" # raise exception if the dfs share a column name if any(df_a.columns.intersection(df_b.columns)): - raise ValueError("Cannot call cross_replicate on dataframes with shared column names") + raise ValueError( + "Cannot call cross_replicate on dataframes with shared column names" + ) size_a, size_b = df_a.shape[0], df_b.shape[0] diff --git a/radp/utility/simulation_utils.py b/radp/utility/simulation_utils.py index c5e255d..ae99e5f 100644 --- a/radp/utility/simulation_utils.py +++ b/radp/utility/simulation_utils.py @@ -8,6 +8,7 @@ import random import torch + def seed_everything(seed: int): random.seed(seed) os.environ["PYTHONHASHSEED"] = str(seed) diff --git a/services/api_manager/app.py b/services/api_manager/app.py index 1a09e82..4370fb7 100644 --- a/services/api_manager/app.py +++ b/services/api_manager/app.py @@ -10,7 +10,9 @@ from api_manager.exceptions.base_api_exception import APIException from api_manager.exceptions.invalid_parameter_exception import InvalidParameterException -from api_manager.handlers.consume_simulation_output_handler import ConsumeSimulationOutputHandler +from api_manager.handlers.consume_simulation_output_handler import ( + ConsumeSimulationOutputHandler, +) from api_manager.handlers.describe_model_handler import DescribeModelHandler from api_manager.handlers.describe_simulation_handler import DescribeSimulationHandler from api_manager.handlers.simulation_handler import SimulationHandler @@ -58,13 +60,17 @@ def train(): # verify request contains payload and csv files if constants.REQUEST_PAYLOAD_FILE_KEY not in request.files: - raise InvalidParameterException(f"Invalid request, missing file input '{constants.REQUEST_PAYLOAD_FILE_KEY}'") + raise InvalidParameterException( + f"Invalid request, missing file input '{constants.REQUEST_PAYLOAD_FILE_KEY}'" + ) if constants.REQUEST_UE_TRAINING_DATA_FILE_KEY not in request.files: raise InvalidParameterException( f"Invalid request, missing file input '{constants.REQUEST_UE_TRAINING_DATA_FILE_KEY}'" ) if constants.REQUEST_TOPOLOGY_FILE_KEY not in request.files: - raise InvalidParameterException(f"Invalid request, missing file input '{constants.REQUEST_TOPOLOGY_FILE_KEY}'") + raise InvalidParameterException( + f"Invalid request, missing file input '{constants.REQUEST_TOPOLOGY_FILE_KEY}'" + ) payload = json.load(request.files[constants.REQUEST_PAYLOAD_FILE_KEY]) @@ -95,7 +101,9 @@ def simulation(): # verify request contains json payload if constants.REQUEST_PAYLOAD_FILE_KEY not in request.files: - raise InvalidParameterException(f"Invalid request, missing file input '{constants.REQUEST_PAYLOAD_FILE_KEY}'") + raise InvalidParameterException( + f"Invalid request, missing file input '{constants.REQUEST_PAYLOAD_FILE_KEY}'" + ) payload = json.load(request.files[constants.REQUEST_PAYLOAD_FILE_KEY]) # store and pass whatever files are provided @@ -118,11 +126,17 @@ def simulation(): @app.route("/simulation/", methods=["GET"]) def describe_simulation(simulation_id: str): logger.info(f"Received API request to describe simulation: {simulation_id}") - return jsonify(DescribeSimulationHandler().handle_describe_simulation_request(simulation_id)) + return jsonify( + DescribeSimulationHandler().handle_describe_simulation_request(simulation_id) + ) @app.route("/simulation//download", methods=["GET"]) def consume_simulation_output(simulation_id: str): logger.info(f"Received API request to consume simulation output: {simulation_id}") - output_zip_file_path = ConsumeSimulationOutputHandler().handle_consume_simulation_output_request(simulation_id) + output_zip_file_path = ( + ConsumeSimulationOutputHandler().handle_consume_simulation_output_request( + simulation_id + ) + ) return send_file(output_zip_file_path) diff --git a/services/api_manager/exceptions/base_api_exception.py b/services/api_manager/exceptions/base_api_exception.py index 79ef4df..9f461f4 100644 --- a/services/api_manager/exceptions/base_api_exception.py +++ b/services/api_manager/exceptions/base_api_exception.py @@ -3,6 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. + class APIException(Exception): """All custom API Exceptions""" diff --git a/services/api_manager/handlers/consume_simulation_output_handler.py b/services/api_manager/handlers/consume_simulation_output_handler.py index 30552d5..9eb3bbe 100644 --- a/services/api_manager/handlers/consume_simulation_output_handler.py +++ b/services/api_manager/handlers/consume_simulation_output_handler.py @@ -14,7 +14,9 @@ import os from api_manager.exceptions.invalid_parameter_exception import InvalidParameterException -from api_manager.exceptions.simulation_output_not_found_exception import SimulationOutputNotFoundException +from api_manager.exceptions.simulation_output_not_found_exception import ( + SimulationOutputNotFoundException, +) from radp.common.helpers.file_system_helper import RADPFileSystemHelper @@ -31,10 +33,14 @@ def handle_consume_simulation_output_request(self, simulation_id: str) -> str: self._validate_request(simulation_id) # get simulation output zipfile - output_zip_file_path = RADPFileSystemHelper.gen_sim_output_zip_file_path(simulation_id) + output_zip_file_path = RADPFileSystemHelper.gen_sim_output_zip_file_path( + simulation_id + ) if not os.path.exists(output_zip_file_path): - logger.warning(f"Unable to find simulation output for simulation: {simulation_id}") + logger.warning( + f"Unable to find simulation output for simulation: {simulation_id}" + ) raise SimulationOutputNotFoundException(simulation_id) logger.info(f"Found output zip file at '{output_zip_file_path}'") diff --git a/services/api_manager/handlers/describe_model_handler.py b/services/api_manager/handlers/describe_model_handler.py index 8df173c..d059c28 100644 --- a/services/api_manager/handlers/describe_model_handler.py +++ b/services/api_manager/handlers/describe_model_handler.py @@ -34,7 +34,9 @@ def handle_describe_model_request(self, model_id: str) -> Dict: try: model_metadata = RADPFileSystemHelper.load_model_metadata(model_id=model_id) except FileNotFoundError: - logger.exception(f"Exception describing model: model '{model_id}' not found") + logger.exception( + f"Exception describing model: model '{model_id}' not found" + ) raise ModelNotFoundException(model_id) # TODO: implement a response DTO to validate response content return model_metadata diff --git a/services/api_manager/handlers/describe_simulation_handler.py b/services/api_manager/handlers/describe_simulation_handler.py index a166ad9..a4cfab7 100644 --- a/services/api_manager/handlers/describe_simulation_handler.py +++ b/services/api_manager/handlers/describe_simulation_handler.py @@ -14,7 +14,9 @@ from typing import Dict from api_manager.exceptions.invalid_parameter_exception import InvalidParameterException -from api_manager.exceptions.simulation_not_found_exception import SimulationNotFoundException +from api_manager.exceptions.simulation_not_found_exception import ( + SimulationNotFoundException, +) from radp.common.helpers.file_system_helper import RADPFileSystemHelper @@ -34,7 +36,9 @@ def handle_describe_simulation_request(self, simulation_id: str) -> Dict: try: sim_metadata = RADPFileSystemHelper.load_simulation_metadata(simulation_id) except FileNotFoundError: - logger.exception(f"Exception describing simulation: simulation '{simulation_id}' not found") + logger.exception( + f"Exception describing simulation: simulation '{simulation_id}' not found" + ) raise SimulationNotFoundException(simulation_id) # TODO: implement a response DTO to validate response content return sim_metadata diff --git a/services/api_manager/handlers/simulation_handler.py b/services/api_manager/handlers/simulation_handler.py index c38b8bd..e48e4be 100644 --- a/services/api_manager/handlers/simulation_handler.py +++ b/services/api_manager/handlers/simulation_handler.py @@ -15,7 +15,9 @@ from api_manager.config.kafka import kafka_producer_config from api_manager.dtos.responses.simulation_response import SimulationResponse -from api_manager.preprocessors.simulation_request_preprocessor import RICSimulationRequestPreprocessor +from api_manager.preprocessors.simulation_request_preprocessor import ( + RICSimulationRequestPreprocessor, +) from confluent_kafka import Producer from radp.common import constants @@ -65,12 +67,16 @@ def handle_simulation_request(self, request: Dict, files: Dict) -> Dict: simulation_id = processed_request[constants.SIMULATION_ID] # create the simulation directory if it doesn't already exist - simulation_directory = RADPFileSystemHelper.gen_simulation_directory(simulation_id) + simulation_directory = RADPFileSystemHelper.gen_simulation_directory( + simulation_id + ) if not os.path.exists(simulation_directory): os.makedirs(simulation_directory) # write simulation metadata to file - RADPFileSystemHelper.save_simulation_metadata(sim_metadata=processed_request, simulation_id=simulation_id) + RADPFileSystemHelper.save_simulation_metadata( + sim_metadata=processed_request, simulation_id=simulation_id + ) # save ue data and config to simulation directory if provided if constants.UE_DATA_FILE_PATH_KEY in files: diff --git a/services/api_manager/handlers/train_handler.py b/services/api_manager/handlers/train_handler.py index c9f8381..e415e0a 100644 --- a/services/api_manager/handlers/train_handler.py +++ b/services/api_manager/handlers/train_handler.py @@ -60,7 +60,9 @@ def handle_train_request(self, request: Dict, files: Dict) -> Dict: topology_file_path=files[constants.TOPOLOGY_FILE_PATH_KEY], ) - model_file_path = RADPFileSystemHelper.gen_model_file_path(train_request.model_id) + model_file_path = RADPFileSystemHelper.gen_model_file_path( + train_request.model_id + ) # create a unique id for the kafka job event key job = { @@ -83,10 +85,7 @@ def handle_train_request(self, request: Dict, files: Dict) -> Dict: ) logger.info(f"Initiated ML training on model {train_request.model_id}.") - return TrainResponse( - job_id=job_id, - model_id=train_request.model_id - ).to_dict() + return TrainResponse(job_id=job_id, model_id=train_request.model_id).to_dict() # TODO: refactor this method, it's gross def _save_metadata_and_topology(self, model_id: str, topology_file_path: str): @@ -97,7 +96,9 @@ def _save_metadata_and_topology(self, model_id: str, topology_file_path: str): topology_df = pd.read_csv(csv_file) num_cells = len(topology_df) except Exception as e: - logger.exception(f"Exception occurred while reading file: {topology_file_path}") + logger.exception( + f"Exception occurred while reading file: {topology_file_path}" + ) raise e model_specific_params = {constants.NUM_CELLS: num_cells} @@ -124,7 +125,9 @@ def _save_metadata_and_topology(self, model_id: str, topology_file_path: str): logger.exception("Exception occurred reading cell topology.csv") raise e - model_topology_file_path = RADPFileSystemHelper.gen_model_topology_file_path(model_id=model_id) + model_topology_file_path = RADPFileSystemHelper.gen_model_topology_file_path( + model_id=model_id + ) write_feather_df(file_path=model_topology_file_path, df=topology_df) def _parse_train_request(self, event) -> TrainRequest: diff --git a/services/api_manager/preprocessors/simulation_request_preprocessor.py b/services/api_manager/preprocessors/simulation_request_preprocessor.py index 59b6896..6fc7c32 100644 --- a/services/api_manager/preprocessors/simulation_request_preprocessor.py +++ b/services/api_manager/preprocessors/simulation_request_preprocessor.py @@ -99,7 +99,11 @@ def preprocess( # start chained hash from top-level simulation parameters chained_hash_val = deterministic_hash_dict( - {constants.SIMULATION_TIME_INTERVAL: request[constants.SIMULATION_TIME_INTERVAL]} + { + constants.SIMULATION_TIME_INTERVAL: request[ + constants.SIMULATION_TIME_INTERVAL + ] + } ) # build processed request frame (this will become to simulation metadata object) @@ -109,7 +113,9 @@ def preprocess( processed_request[constants.SIMULATION_STATUS] = constants.STATUS_PLANNED # supply simulation interval value - processed_request[constants.SIMULATION_TIME_INTERVAL] = request[constants.SIMULATION_TIME_INTERVAL] + processed_request[constants.SIMULATION_TIME_INTERVAL] = request[ + constants.SIMULATION_TIME_INTERVAL + ] # get the ue_tracks hash value chained_hash_val = deterministic_hash_dict( @@ -124,27 +130,32 @@ def preprocess( if constants.UE_TRACKS_GENERATION in request[constants.UE_TRACKS]: # supply UE tracks generation object to the processed request processed_request[constants.UE_TRACKS_GENERATION] = {} - processed_request[constants.UE_TRACKS_GENERATION][constants.PARAMS] = request[constants.UE_TRACKS][ - constants.UE_TRACKS_GENERATION - ] + processed_request[constants.UE_TRACKS_GENERATION][ + constants.PARAMS + ] = request[constants.UE_TRACKS][constants.UE_TRACKS_GENERATION] # get num_ticks using division of duration by time interval - simulation_duration = processed_request[constants.UE_TRACKS_GENERATION][constants.PARAMS][ - constants.SIMULATION_DURATION - ] + simulation_duration = processed_request[constants.UE_TRACKS_GENERATION][ + constants.PARAMS + ][constants.SIMULATION_DURATION] processed_request[constants.NUM_TICKS] = int( - simulation_duration / processed_request[constants.SIMULATION_TIME_INTERVAL] + simulation_duration + / processed_request[constants.SIMULATION_TIME_INTERVAL] ) # set hash value in ue_tracks generation - processed_request[constants.UE_TRACKS_GENERATION][constants.HASH_VAL] = chained_hash_val + processed_request[constants.UE_TRACKS_GENERATION][ + constants.HASH_VAL + ] = chained_hash_val # add state object to UE tracks generation object processed_request[constants.UE_TRACKS_GENERATION][constants.STATE] = {} processed_request[constants.UE_TRACKS_GENERATION][constants.STATE][ constants.STATUS ] = constants.STATUS_PLANNED - processed_request[constants.UE_TRACKS_GENERATION][constants.STATE][constants.BATCHES_OUTPUTTED] = 0 + processed_request[constants.UE_TRACKS_GENERATION][constants.STATE][ + constants.BATCHES_OUTPUTTED + ] = 0 else: # TODO: get actual num_ticks value by scanning input file "tick" column @@ -173,19 +184,29 @@ def _preprocess_stage( # initial empty state processed_request[stage][constants.STATE] = {} - processed_request[stage][constants.STATE][constants.STATUS] = constants.STATUS_PLANNED - processed_request[stage][constants.STATE][constants.LATEST_BATCH_WITHOUT_FAILURE] = 0 - processed_request[stage][constants.STATE][constants.LATEST_BATCH_TO_SUCCEED] = 0 + processed_request[stage][constants.STATE][ + constants.STATUS + ] = constants.STATUS_PLANNED + processed_request[stage][constants.STATE][ + constants.LATEST_BATCH_WITHOUT_FAILURE + ] = 0 + processed_request[stage][constants.STATE][ + constants.LATEST_BATCH_TO_SUCCEED + ] = 0 processed_request[stage][constants.STATE][constants.BATCHES_RETRYING] = [] return chained_hash_val # RF Prediction if constants.RF_PREDICTION in request: - chained_hash_val = _preprocess_stage(stage=constants.RF_PREDICTION, chained_hash_val=chained_hash_val) + chained_hash_val = _preprocess_stage( + stage=constants.RF_PREDICTION, chained_hash_val=chained_hash_val + ) # Protocol Emulation if constants.PROTOCOL_EMULATION in request: - chained_hash_val = _preprocess_stage(stage=constants.PROTOCOL_EMULATION, chained_hash_val=chained_hash_val) + chained_hash_val = _preprocess_stage( + stage=constants.PROTOCOL_EMULATION, chained_hash_val=chained_hash_val + ) # Simulation ID def _create_simulation_id( @@ -194,24 +215,28 @@ def _create_simulation_id( """Create the simulation ID from hashing everything together""" # pull the non-stage-specific fields constituent_hashes = { - k: processed_request[k] for k in (constants.NUM_TICKS, constants.NUM_BATCHES) if k in processed_request + k: processed_request[k] + for k in (constants.NUM_TICKS, constants.NUM_BATCHES) + if k in processed_request } # pull all present stage hashes if constants.UE_TRACKS_GENERATION in processed_request: - constituent_hashes[constants.UE_TRACKS_GENERATION_HASH_VAL] = processed_request[ - constants.UE_TRACKS_GENERATION - ][constants.HASH_VAL] - - if constants.RF_PREDICTION in processed_request: - constituent_hashes[constants.RF_PREDICTION_HASH_VAL] = processed_request[constants.RF_PREDICTION][ + constituent_hashes[ + constants.UE_TRACKS_GENERATION_HASH_VAL + ] = processed_request[constants.UE_TRACKS_GENERATION][ constants.HASH_VAL ] + if constants.RF_PREDICTION in processed_request: + constituent_hashes[ + constants.RF_PREDICTION_HASH_VAL + ] = processed_request[constants.RF_PREDICTION][constants.HASH_VAL] + if constants.PROTOCOL_EMULATION in processed_request: - constituent_hashes[constants.PROTOCOL_EMULATION_HASH_VAL] = processed_request[ - constants.PROTOCOL_EMULATION - ][constants.HASH_VAL] + constituent_hashes[ + constants.PROTOCOL_EMULATION_HASH_VAL + ] = processed_request[constants.PROTOCOL_EMULATION][constants.HASH_VAL] # build the one hash # one hash to rule them all... throw it into the fire you fool! diff --git a/services/api_manager/tests/handlers/test_consume_simulation_output_handler.py b/services/api_manager/tests/handlers/test_consume_simulation_output_handler.py index 3e7b0ba..cf1d658 100644 --- a/services/api_manager/tests/handlers/test_consume_simulation_output_handler.py +++ b/services/api_manager/tests/handlers/test_consume_simulation_output_handler.py @@ -7,36 +7,58 @@ from unittest.mock import MagicMock, patch from api_manager.exceptions.invalid_parameter_exception import InvalidParameterException -from api_manager.exceptions.simulation_output_not_found_exception import SimulationOutputNotFoundException -from api_manager.handlers.consume_simulation_output_handler import ConsumeSimulationOutputHandler +from api_manager.exceptions.simulation_output_not_found_exception import ( + SimulationOutputNotFoundException, +) +from api_manager.handlers.consume_simulation_output_handler import ( + ConsumeSimulationOutputHandler, +) class TestConsumeSimulationOutputHandler(TestCase): @patch("api_manager.handlers.consume_simulation_output_handler.os") - @patch("api_manager.handlers.consume_simulation_output_handler.RADPFileSystemHelper") - def test_handle_consume_simulation_output_request(self, mock_file_system_helper: MagicMock, mock_os: MagicMock): - mock_file_system_helper.gen_sim_output_zip_file_path.return_value = "dummy_outzip_zip" + @patch( + "api_manager.handlers.consume_simulation_output_handler.RADPFileSystemHelper" + ) + def test_handle_consume_simulation_output_request( + self, mock_file_system_helper: MagicMock, mock_os: MagicMock + ): + mock_file_system_helper.gen_sim_output_zip_file_path.return_value = ( + "dummy_outzip_zip" + ) mock_os.path.exists.return_value = True dummy_sim_id = "dummy_sim" assert ( - ConsumeSimulationOutputHandler().handle_consume_simulation_output_request(dummy_sim_id) + ConsumeSimulationOutputHandler().handle_consume_simulation_output_request( + dummy_sim_id + ) == "dummy_outzip_zip" ) - mock_file_system_helper.gen_sim_output_zip_file_path.assert_called_once_with("dummy_sim") + mock_file_system_helper.gen_sim_output_zip_file_path.assert_called_once_with( + "dummy_sim" + ) mock_os.path.exists.assert_called_once_with("dummy_outzip_zip") @patch("api_manager.handlers.consume_simulation_output_handler.os") - @patch("api_manager.handlers.consume_simulation_output_handler.RADPFileSystemHelper") + @patch( + "api_manager.handlers.consume_simulation_output_handler.RADPFileSystemHelper" + ) def test_handle_consume_simulation_output_request__nonexistent_model( self, mock_file_system_helper: MagicMock, mock_os: MagicMock ): - mock_file_system_helper.gen_sim_output_zip_file_path.return_value = "dummy_outzip_zip" + mock_file_system_helper.gen_sim_output_zip_file_path.return_value = ( + "dummy_outzip_zip" + ) mock_os.path.exists.return_value = False with self.assertRaises(SimulationOutputNotFoundException): - assert ConsumeSimulationOutputHandler().handle_consume_simulation_output_request("dummy_sim") + assert ConsumeSimulationOutputHandler().handle_consume_simulation_output_request( + "dummy_sim" + ) def test_handle_consume_simulation_output_request__no_sim_id(self): with self.assertRaises(InvalidParameterException): - ConsumeSimulationOutputHandler().handle_consume_simulation_output_request("") + ConsumeSimulationOutputHandler().handle_consume_simulation_output_request( + "" + ) diff --git a/services/api_manager/tests/handlers/test_describe_model_handler.py b/services/api_manager/tests/handlers/test_describe_model_handler.py index cadad06..4ebc6e5 100644 --- a/services/api_manager/tests/handlers/test_describe_model_handler.py +++ b/services/api_manager/tests/handlers/test_describe_model_handler.py @@ -15,11 +15,17 @@ class TestDescribeModelHandler(TestCase): @patch("api_manager.handlers.describe_model_handler.RADPFileSystemHelper") def test_handle_describe_model_request(self, mock_file_system_helper: MagicMock): mock_file_system_helper.load_model_metadata.return_value = {"metadata": "dummy"} - assert DescribeModelHandler().handle_describe_model_request("dummy_model") == {"metadata": "dummy"} - mock_file_system_helper.load_model_metadata.assert_called_once_with(model_id="dummy_model") + assert DescribeModelHandler().handle_describe_model_request("dummy_model") == { + "metadata": "dummy" + } + mock_file_system_helper.load_model_metadata.assert_called_once_with( + model_id="dummy_model" + ) @patch("api_manager.handlers.describe_model_handler.RADPFileSystemHelper") - def test_handle_describe_model_request__model_not_found(self, mock_file_system_helper: MagicMock): + def test_handle_describe_model_request__model_not_found( + self, mock_file_system_helper: MagicMock + ): mock_file_system_helper.side_effect mock_file_system_helper.load_model_metadata.side_effect = FileNotFoundError() with self.assertRaises(ModelNotFoundException): diff --git a/services/api_manager/tests/handlers/test_describe_simulation_handler.py b/services/api_manager/tests/handlers/test_describe_simulation_handler.py index dfb7147..28ef8df 100644 --- a/services/api_manager/tests/handlers/test_describe_simulation_handler.py +++ b/services/api_manager/tests/handlers/test_describe_simulation_handler.py @@ -7,25 +7,39 @@ from unittest.mock import MagicMock, patch from api_manager.exceptions.invalid_parameter_exception import InvalidParameterException -from api_manager.exceptions.simulation_not_found_exception import SimulationNotFoundException +from api_manager.exceptions.simulation_not_found_exception import ( + SimulationNotFoundException, +) from api_manager.handlers.describe_simulation_handler import DescribeSimulationHandler class TestDescribeSimulationHandler(TestCase): @patch("api_manager.handlers.describe_simulation_handler.RADPFileSystemHelper") - def test_handle_describe_simulation_request(self, mock_file_system_helper: MagicMock): - mock_file_system_helper.load_simulation_metadata.return_value = {"metadata": "dummy"} - assert DescribeSimulationHandler().handle_describe_simulation_request("dummy_simulation") == { + def test_handle_describe_simulation_request( + self, mock_file_system_helper: MagicMock + ): + mock_file_system_helper.load_simulation_metadata.return_value = { "metadata": "dummy" } - mock_file_system_helper.load_simulation_metadata.assert_called_once_with("dummy_simulation") + assert DescribeSimulationHandler().handle_describe_simulation_request( + "dummy_simulation" + ) == {"metadata": "dummy"} + mock_file_system_helper.load_simulation_metadata.assert_called_once_with( + "dummy_simulation" + ) @patch("api_manager.handlers.describe_simulation_handler.RADPFileSystemHelper") - def test_handle_describe_simulation_request__simulation_not_found(self, mock_file_system_helper: MagicMock): + def test_handle_describe_simulation_request__simulation_not_found( + self, mock_file_system_helper: MagicMock + ): mock_file_system_helper.side_effect - mock_file_system_helper.load_simulation_metadata.side_effect = FileNotFoundError() + mock_file_system_helper.load_simulation_metadata.side_effect = ( + FileNotFoundError() + ) with self.assertRaises(SimulationNotFoundException): - DescribeSimulationHandler().handle_describe_simulation_request("dummy_simulation") + DescribeSimulationHandler().handle_describe_simulation_request( + "dummy_simulation" + ) def test_handle_describe_simulation_request__invalid_simulation_id(self): with self.assertRaises(InvalidParameterException): diff --git a/services/api_manager/tests/handlers/test_simulation_handler.py b/services/api_manager/tests/handlers/test_simulation_handler.py index 767cf47..5142553 100644 --- a/services/api_manager/tests/handlers/test_simulation_handler.py +++ b/services/api_manager/tests/handlers/test_simulation_handler.py @@ -42,12 +42,11 @@ def test_handle_simulation_request( "simulation_id": "dummy_sim_id", } - expected_job = { - "job_type": "orchestration", - "simulation_id": "dummy_sim_id" - } + expected_job = {"job_type": "orchestration", "simulation_id": "dummy_sim_id"} - assert SimulationHandler().handle_simulation_request(dummy_rf_sim, dummy_files) == { + assert SimulationHandler().handle_simulation_request( + dummy_rf_sim, dummy_files + ) == { "job_id": "dummy_job_id", "simulation_id": "dummy_sim_id", } @@ -60,4 +59,6 @@ def test_handle_simulation_request( "dummy_sim_id", config_file_path="dummy_config_file_path" ) - mock_produce.assert_called_once_with(producer=mock_producer_instance, topic="jobs", value=expected_job) + mock_produce.assert_called_once_with( + producer=mock_producer_instance, topic="jobs", value=expected_job + ) diff --git a/services/api_manager/tests/handlers/test_train_handler.py b/services/api_manager/tests/handlers/test_train_handler.py index 76d152e..01a9c60 100644 --- a/services/api_manager/tests/handlers/test_train_handler.py +++ b/services/api_manager/tests/handlers/test_train_handler.py @@ -57,7 +57,9 @@ def test_handle_train_request__missing_model_id( ): mock_producer_instance = MagicMock mock_producer.return_value = mock_producer_instance - mock_file_system_helper.gen_model_file_path.return_value = "dummy_model_file_path" + mock_file_system_helper.gen_model_file_path.return_value = ( + "dummy_model_file_path" + ) mock_produce.return_value = "dummy_job_id" @@ -75,13 +77,19 @@ def test_handle_train_request__missing_model_id( "topology_file_path": "dummy_topology_file_path", } - assert TrainHandler().handle_train_request(valid_train_request_1, files=dummy_files) == { + assert TrainHandler().handle_train_request( + valid_train_request_1, files=dummy_files + ) == { "job_id": "dummy_job_id", "model_id": "dummy_model", } - mock_file_system_helper.gen_model_file_path.assert_called_once_with("dummy_model") + mock_file_system_helper.gen_model_file_path.assert_called_once_with( + "dummy_model" + ) mock_file_system_helper.save_model_metadata.assert_called_once() mock_file_system_helper.gen_model_topology_file_path.assert_called_once() - mock_produce.assert_called_once_with(producer=mock_producer_instance, topic="jobs", value=expected_job) + mock_produce.assert_called_once_with( + producer=mock_producer_instance, topic="jobs", value=expected_job + ) diff --git a/services/api_manager/tests/preprocessors/test_simulation_request_preprocessor.py b/services/api_manager/tests/preprocessors/test_simulation_request_preprocessor.py index 5eeb6e5..eb3b5b3 100644 --- a/services/api_manager/tests/preprocessors/test_simulation_request_preprocessor.py +++ b/services/api_manager/tests/preprocessors/test_simulation_request_preprocessor.py @@ -99,7 +99,11 @@ def _test_ue_tracks( ue_tracks_specifier_key: str, ): chained_hash_val = deterministic_hash_dict( - {"simulation_time_interval_seconds": request["simulation_time_interval_seconds"]} + { + "simulation_time_interval_seconds": request[ + "simulation_time_interval_seconds" + ] + } ) self.assertEqual( diff --git a/services/api_manager/tests/validators/test_simulation_request_validator.py b/services/api_manager/tests/validators/test_simulation_request_validator.py index 2675ae9..a5bc49c 100644 --- a/services/api_manager/tests/validators/test_simulation_request_validator.py +++ b/services/api_manager/tests/validators/test_simulation_request_validator.py @@ -6,7 +6,9 @@ import unittest from api_manager.exceptions.invalid_parameter_exception import InvalidParameterException -from api_manager.validators.simulation_request_validator import RICSimulationRequestValidator +from api_manager.validators.simulation_request_validator import ( + RICSimulationRequestValidator, +) class TestRICSimulationRequestValidator(unittest.TestCase): @@ -31,13 +33,17 @@ def test__validate_rf_prediction(self): # Missing rf_prediction (but ue_tracks present and valid) with self.assertRaises(InvalidParameterException) as ipe: - RICSimulationRequestValidator._validate_rf_prediction({"ue_tracks": {"ue_tracks_generation": {}}}) + RICSimulationRequestValidator._validate_rf_prediction( + {"ue_tracks": {"ue_tracks_generation": {}}} + ) self.assertEqual( str(ipe.exception), "Missing rf_prediction key in RIC Simulation Request spec!", ) with self.assertRaises(InvalidParameterException) as ipe: - RICSimulationRequestValidator._validate_rf_prediction({"ue_tracks": {"ue_data_id": {}}}) + RICSimulationRequestValidator._validate_rf_prediction( + {"ue_tracks": {"ue_data_id": {}}} + ) self.assertEqual( str(ipe.exception), "Missing rf_prediction key in RIC Simulation Request spec!", diff --git a/services/api_manager/utils/file_io.py b/services/api_manager/utils/file_io.py index d0f13fc..c21edea 100644 --- a/services/api_manager/utils/file_io.py +++ b/services/api_manager/utils/file_io.py @@ -22,7 +22,9 @@ def get_utc_timestamp() -> str: return now_utc.strftime("%Y_%m_%d-%I_%M_%S_%p") -def save_file_from_flask(file_storage: FileStorage, upload_folder: str, file_name: str) -> str: +def save_file_from_flask( + file_storage: FileStorage, upload_folder: str, file_name: str +) -> str: """Helper method to save a werkzeug FileStorage file to disk""" try: # create folder if it does not exist @@ -66,12 +68,16 @@ def bootstrap_radp_filesystem(): try: os.makedirs(directory) except Exception as e: - logger.exception(f"Exception occurred creating directory '{directory}': {e}") + logger.exception( + f"Exception occurred creating directory '{directory}': {e}" + ) raise e directories_output_string = "\n".join(SYSTEM_DIRECTORIES) logger.info( - "Successfully created the following directories:\n{directories}".format(directories=directories_output_string) + "Successfully created the following directories:\n{directories}".format( + directories=directories_output_string + ) ) diff --git a/services/api_manager/validators/simulation_request_validator.py b/services/api_manager/validators/simulation_request_validator.py index 72c32d5..8280644 100644 --- a/services/api_manager/validators/simulation_request_validator.py +++ b/services/api_manager/validators/simulation_request_validator.py @@ -100,7 +100,8 @@ def _validate_rf_prediction(request: Dict): """ if ("ue_tracks" not in request) or ( - ("ue_tracks_generation" not in request["ue_tracks"]) and ("ue_data_id" not in request["ue_tracks"]) + ("ue_tracks_generation" not in request["ue_tracks"]) + and ("ue_data_id" not in request["ue_tracks"]) ): raise InvalidParameterException( "Must provide ue_tracks section with either provide `ue_tracks_generation` " @@ -108,6 +109,8 @@ def _validate_rf_prediction(request: Dict): ) if "rf_prediction" not in request: - raise InvalidParameterException("Missing rf_prediction key in RIC Simulation Request spec!") + raise InvalidParameterException( + "Missing rf_prediction key in RIC Simulation Request spec!" + ) return None diff --git a/services/orchestration/orchestration_consumer.py b/services/orchestration/orchestration_consumer.py index e2e23f3..c0bcec0 100644 --- a/services/orchestration/orchestration_consumer.py +++ b/services/orchestration/orchestration_consumer.py @@ -47,11 +47,15 @@ def consume(self): logger.debug("Waiting...") continue if message.error(): - logger.exception(f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}") + logger.exception( + f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}" + ) continue # Extract the (optional) key and value, and print. - logger.debug(f"Consumed message value = {message.value().decode('utf-8')}") + logger.debug( + f"Consumed message value = {message.value().decode('utf-8')}" + ) # pull event object from message event = json.loads(message.value().decode("utf-8")) @@ -60,12 +64,19 @@ def consume(self): try: # check which topic message is from if message.topic() == constants.KAFKA_JOBS_TOPIC_NAME: - if event[constants.KAFKA_JOB_TYPE] != constants.JOB_TYPE_ORCHESTRATION: + if ( + event[constants.KAFKA_JOB_TYPE] + != constants.JOB_TYPE_ORCHESTRATION + ): # skip non-orchestration jobs - logger.debug(f"Consumed non-orchestration job: {event}... skipping") + logger.debug( + f"Consumed non-orchestration job: {event}... skipping" + ) else: # handle orchestration job - logger.info(f"Consumed orchestration job: {event}... handling") + logger.info( + f"Consumed orchestration job: {event}... handling" + ) self.orchestrator.handle_orchestration_job(event) continue else: diff --git a/services/orchestration/orchestration_helper.py b/services/orchestration/orchestration_helper.py index d4b0fa0..42e4087 100644 --- a/services/orchestration/orchestration_helper.py +++ b/services/orchestration/orchestration_helper.py @@ -70,7 +70,9 @@ def get_output_stage(sim_metadata: Dict) -> SimulationStage: @staticmethod def get_rf_digital_twin_model_id(sim_metadata: Dict) -> str: """Get the RF digital twin model used in simulation""" - return sim_metadata[SimulationStage.RF_PREDICTION.value][constants.PARAMS][constants.MODEL_ID] + return sim_metadata[SimulationStage.RF_PREDICTION.value][constants.PARAMS][ + constants.MODEL_ID + ] @staticmethod def has_stage(sim_metadata: Dict, stage: SimulationStage) -> bool: @@ -90,7 +92,9 @@ def get_stage_hash_val(sim_metadata: Dict, stage: SimulationStage) -> str: return sim_metadata[stage.value][constants.HASH_VAL] @staticmethod - def generate_job_event_frame(sim_metadata: Dict, stage: SimulationStage, batch=None) -> Dict: + def generate_job_event_frame( + sim_metadata: Dict, stage: SimulationStage, batch=None + ) -> Dict: """Generate a standard frame for job event given a stage""" job_frame: Dict[str, Any] = {} @@ -110,16 +114,28 @@ def stage_has_completed(sim_metadata: Dict, stage: SimulationStage): """Check if the stage has completed""" _, num_batches = OrchestrationHelper.get_batching_params(sim_metadata) if stage == SimulationStage.UE_TRACKS_GENERATION: - ue_tracks_state = sim_metadata[SimulationStage.UE_TRACKS_GENERATION.value][constants.STATE] + ue_tracks_state = sim_metadata[SimulationStage.UE_TRACKS_GENERATION.value][ + constants.STATE + ] return ue_tracks_state[constants.BATCHES_OUTPUTTED] == num_batches if stage == SimulationStage.RF_PREDICTION: - rf_prediction_state = sim_metadata[SimulationStage.RF_PREDICTION.value][constants.STATE] - return rf_prediction_state[constants.LATEST_BATCH_WITHOUT_FAILURE] == num_batches + rf_prediction_state = sim_metadata[SimulationStage.RF_PREDICTION.value][ + constants.STATE + ] + return ( + rf_prediction_state[constants.LATEST_BATCH_WITHOUT_FAILURE] + == num_batches + ) if stage == SimulationStage.PROTOCOL_EMULATION: - protocol_emulation_state = sim_metadata[SimulationStage.PROTOCOL_EMULATION.value][constants.STATE] - return protocol_emulation_state[constants.LATEST_BATCH_WITHOUT_FAILURE] == num_batches + protocol_emulation_state = sim_metadata[ + SimulationStage.PROTOCOL_EMULATION.value + ][constants.STATE] + return ( + protocol_emulation_state[constants.LATEST_BATCH_WITHOUT_FAILURE] + == num_batches + ) else: logger.exception("Received unexpected stage: {stage.value}") raise ValueError("Received unexpected stage: {stage.value}") diff --git a/services/orchestration/orchestrator.py b/services/orchestration/orchestrator.py index 75c2faa..4161469 100644 --- a/services/orchestration/orchestrator.py +++ b/services/orchestration/orchestrator.py @@ -142,7 +142,9 @@ def handle_orchestration_job(self, job_data: Dict): # input data must be in cache if the first non-cached stage is not the first present stage input_data_is_cached = first_present_stage != first_non_cached_stage - logger.info(f"Running first non-cached stage: {first_non_cached_stage.name}") + logger.info( + f"Running first non-cached stage: {first_non_cached_stage.name}" + ) self._run_first_non_cached_stage( sim_metadata, simulation_id, @@ -151,7 +153,9 @@ def handle_orchestration_job(self, job_data: Dict): ) sim_metadata[constants.JOB_ID] = job_id - sim_metadata[constants.JOB_FINISHED_DATETIME] = datetime.datetime.now(datetime.timezone.utc).isoformat() + sim_metadata[constants.JOB_FINISHED_DATETIME] = datetime.datetime.now( + datetime.timezone.utc + ).isoformat() # save state after orchestration RADPFileSystemHelper.save_simulation_metadata(sim_metadata, simulation_id) @@ -169,11 +173,15 @@ def handle_output_event(self, output_event: Dict): output_service = output_event[constants.SERVICE] if output_service == SimulationStage.UE_TRACKS_GENERATION.value: - self._handle_ue_tracks_generation_output(sim_metadata, output_event, simulation_id) + self._handle_ue_tracks_generation_output( + sim_metadata, output_event, simulation_id + ) elif output_service == SimulationStage.RF_PREDICTION.value: self._handle_rf_prediction_output(sim_metadata, output_event, simulation_id) else: - self._handle_protocol_emulation_output(sim_metadata, output_event, simulation_id) + self._handle_protocol_emulation_output( + sim_metadata, output_event, simulation_id + ) # save state after handling output RADPFileSystemHelper.save_simulation_metadata(sim_metadata, simulation_id) @@ -213,7 +221,10 @@ def _run_first_non_cached_stage( if first_non_cached_stage == SimulationStage.UE_TRACKS_GENERATION: # stage is UE Tracks Generation --> no input data, data is generated return self._start_ue_tracks_generation(sim_metadata, simulation_id) - elif first_non_cached_stage == SimulationStage.RF_PREDICTION and input_data_is_cached: + elif ( + first_non_cached_stage == SimulationStage.RF_PREDICTION + and input_data_is_cached + ): # stage is RF Prediction + input data is cached # --> start all jobs (which pulls from cache) return self._start_all_rf_prediction_jobs(sim_metadata, simulation_id) @@ -223,7 +234,10 @@ def _run_first_non_cached_stage( return self._start_single_rf_prediction_job( sim_metadata, simulation_id, batch=1, data_source=DataSource.USER_INPUT ) - elif first_non_cached_stage == SimulationStage.PROTOCOL_EMULATION and input_data_is_cached: + elif ( + first_non_cached_stage == SimulationStage.PROTOCOL_EMULATION + and input_data_is_cached + ): # stage is RF Prediction + input data is cached # --> start all jobs (which pulls from cache) return self._start_all_protocol_emulation_jobs(sim_metadata, simulation_id) @@ -269,7 +283,9 @@ def _start_ue_tracks_generation(self, sim_metadata: Dict, simulation_id: str): # pull the job parameters from the simulation metadata ue_tracks_params = {} ue_tracks_params.update( - OrchestrationHelper.get_stage_params(sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION) + OrchestrationHelper.get_stage_params( + sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION + ) ) # pull the interval @@ -279,7 +295,9 @@ def _start_ue_tracks_generation(self, sim_metadata: Dict, simulation_id: str): num_ticks, num_batches = OrchestrationHelper.get_batching_params(sim_metadata) # pull hash_val and build output file prefix - hash_val = OrchestrationHelper.get_stage_hash_val(sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION) + hash_val = OrchestrationHelper.get_stage_hash_val( + sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION + ) output_file_prefix = f"{SimulationStage.UE_TRACKS_GENERATION.value}-{hash_val}" # build the ue tracks generation job event @@ -288,7 +306,9 @@ def _start_ue_tracks_generation(self, sim_metadata: Dict, simulation_id: str): ue_tracks_params[constants.NUM_BATCHES] = num_batches # generate the kafka job frame - ue_tracks_job = OrchestrationHelper.generate_job_event_frame(sim_metadata, SimulationStage.UE_TRACKS_GENERATION) + ue_tracks_job = OrchestrationHelper.generate_job_event_frame( + sim_metadata, SimulationStage.UE_TRACKS_GENERATION + ) # supply stage-specific fields ue_tracks_job[SimulationStage.UE_TRACKS_GENERATION.value].update( @@ -299,11 +319,17 @@ def _start_ue_tracks_generation(self, sim_metadata: Dict, simulation_id: str): ) # produce ue_tracks_generation job to jobs topic - produce_object_to_kafka_topic(self.producer, topic=constants.KAFKA_JOBS_TOPIC_NAME, value=ue_tracks_job) - logger.info(f"Produced UE tracks generation job for simulation: {simulation_id}") + produce_object_to_kafka_topic( + self.producer, topic=constants.KAFKA_JOBS_TOPIC_NAME, value=ue_tracks_job + ) + logger.info( + f"Produced UE tracks generation job for simulation: {simulation_id}" + ) # Update state to show UE tracks generation has started - ue_tracks_state = sim_metadata[SimulationStage.UE_TRACKS_GENERATION.value][constants.STATE] + ue_tracks_state = sim_metadata[SimulationStage.UE_TRACKS_GENERATION.value][ + constants.STATE + ] ue_tracks_state[constants.STATUS] = WorkflowStatus.IN_PROGRESS.value def _start_all_rf_prediction_jobs(self, sim_metadata: Dict, simulation_id: str): @@ -311,9 +337,13 @@ def _start_all_rf_prediction_jobs(self, sim_metadata: Dict, simulation_id: str): # pull number of batches to start _, num_batches = OrchestrationHelper.get_batching_params(sim_metadata) - logger.info(f"Starting all {num_batches} RF Prediction jobs for simulation: {simulation_id}") + logger.info( + f"Starting all {num_batches} RF Prediction jobs for simulation: {simulation_id}" + ) for i in range(1, num_batches + 1): - self._start_single_rf_prediction_job(sim_metadata, simulation_id, batch=i, data_source=DataSource.CACHE) + self._start_single_rf_prediction_job( + sim_metadata, simulation_id, batch=i, data_source=DataSource.CACHE + ) def _start_single_rf_prediction_job( self, @@ -345,7 +375,9 @@ def _start_single_rf_prediction_job( # check if input is from simulation folder or UE tracks generation output layer if data_source == DataSource.USER_INPUT: # input is in simulation folder, get its path - ue_data_file_path = RADPFileSystemHelper.gen_simulation_ue_data_file_path(simulation_id) + ue_data_file_path = RADPFileSystemHelper.gen_simulation_ue_data_file_path( + simulation_id + ) else: # input is cached in UE tracks generation output layer, get its path using its hash ue_tracks_hash_val = OrchestrationHelper.get_stage_hash_val( @@ -358,7 +390,9 @@ def _start_single_rf_prediction_job( ) # get output file path using the RF Prediction hash - rf_prediction_hash_val = OrchestrationHelper.get_stage_hash_val(sim_metadata, SimulationStage.RF_PREDICTION) + rf_prediction_hash_val = OrchestrationHelper.get_stage_hash_val( + sim_metadata, SimulationStage.RF_PREDICTION + ) output_file_path = RADPFileSystemHelper.gen_stage_output_file_path( stage=SimulationStage.RF_PREDICTION, hash_val=rf_prediction_hash_val, @@ -370,11 +404,15 @@ def _start_single_rf_prediction_job( model_file_path = RADPFileSystemHelper.gen_model_file_path(model_id) # get the config and topology file paths - config_file_path = RADPFileSystemHelper.gen_simulation_cell_config_file_path(simulation_id) + config_file_path = RADPFileSystemHelper.gen_simulation_cell_config_file_path( + simulation_id + ) topology_file_path = RADPFileSystemHelper.gen_model_topology_file_path(model_id) # get the RF prediction params - rf_prediction_params = OrchestrationHelper.get_stage_params(sim_metadata, stage=SimulationStage.RF_PREDICTION) + rf_prediction_params = OrchestrationHelper.get_stage_params( + sim_metadata, stage=SimulationStage.RF_PREDICTION + ) # build the rf prediction job rf_prediction_job = OrchestrationHelper.generate_job_event_frame( @@ -398,11 +436,15 @@ def _start_single_rf_prediction_job( topic=constants.KAFKA_JOBS_TOPIC_NAME, value=rf_prediction_job, ) - logger.info(f"Produced RF Prediction job batch {batch} for simulation: {simulation_id}") + logger.info( + f"Produced RF Prediction job batch {batch} for simulation: {simulation_id}" + ) # Update state to show RF Prediction has started if batch == 1: - rf_prediction_state = sim_metadata[SimulationStage.RF_PREDICTION.value][constants.STATE] + rf_prediction_state = sim_metadata[SimulationStage.RF_PREDICTION.value][ + constants.STATE + ] rf_prediction_state[constants.STATUS] = WorkflowStatus.IN_PROGRESS.value def _start_all_protocol_emulation_jobs( @@ -425,28 +467,42 @@ def _start_single_protocol_emulation_job( # TODO: implement this once protocol emulation service is implemented pass - def _handle_ue_tracks_generation_output(self, sim_metadata: Dict, ue_tracks_output: Dict, simulation_id: str): + def _handle_ue_tracks_generation_output( + self, sim_metadata: Dict, ue_tracks_output: Dict, simulation_id: str + ): """Handle a UE tracks generation output""" # TODO: handle a failed job. We'll probably want to add the original job event to the output object # to make retry easier if ue_tracks_output[constants.STATUS] == OutputStatus.FAILURE.value: - logger.exception(f"Simulation {simulation_id} failed in UE tracks generation stage: {ue_tracks_output}") - raise Exception(f"Simulation {simulation_id} failed in UE tracks generation stage: {ue_tracks_output}") + logger.exception( + f"Simulation {simulation_id} failed in UE tracks generation stage: {ue_tracks_output}" + ) + raise Exception( + f"Simulation {simulation_id} failed in UE tracks generation stage: {ue_tracks_output}" + ) # pull state and batch - ue_tracks_state = sim_metadata[SimulationStage.UE_TRACKS_GENERATION.value][constants.STATE] + ue_tracks_state = sim_metadata[SimulationStage.UE_TRACKS_GENERATION.value][ + constants.STATE + ] batch = ue_tracks_output[constants.BATCH] # update the outputted batches field ue_tracks_state[constants.BATCHES_OUTPUTTED] = batch # check if the stage has completed, update stage status if so - stage_completed = OrchestrationHelper.stage_has_completed(sim_metadata, SimulationStage.UE_TRACKS_GENERATION) + stage_completed = OrchestrationHelper.stage_has_completed( + sim_metadata, SimulationStage.UE_TRACKS_GENERATION + ) if stage_completed: ue_tracks_state[constants.STATUS] = WorkflowStatus.FINISHED.value - if OrchestrationHelper.has_stage(sim_metadata, stage=SimulationStage.RF_PREDICTION): - self._start_single_rf_prediction_job(sim_metadata, simulation_id, batch, data_source=DataSource.CACHE) + if OrchestrationHelper.has_stage( + sim_metadata, stage=SimulationStage.RF_PREDICTION + ): + self._start_single_rf_prediction_job( + sim_metadata, simulation_id, batch, data_source=DataSource.CACHE + ) elif stage_completed: # this was the last stage, move to wrap up self._wrap_up_simulation(sim_metadata, simulation_id) @@ -455,16 +511,24 @@ def _handle_ue_tracks_generation_output(self, sim_metadata: Dict, ue_tracks_outp # save simulation metadata and exit pass - def _handle_rf_prediction_output(self, sim_metadata: Dict, rf_prediction_output: Dict, simulation_id: str): + def _handle_rf_prediction_output( + self, sim_metadata: Dict, rf_prediction_output: Dict, simulation_id: str + ): """Handle an RF prediction output""" # TODO: handle a failed job. We'll probably want to add the original job event to the output object # to make retry easier if rf_prediction_output[constants.STATUS] == OutputStatus.FAILURE.value: - logger.exception(f"Simulation {simulation_id} failed in RF Prediction stage: {rf_prediction_output}") - raise Exception(f"Simulation {simulation_id} failed in RF Prediction stage: {rf_prediction_output}") + logger.exception( + f"Simulation {simulation_id} failed in RF Prediction stage: {rf_prediction_output}" + ) + raise Exception( + f"Simulation {simulation_id} failed in RF Prediction stage: {rf_prediction_output}" + ) # pull state and batch - rf_prediction_state = sim_metadata[SimulationStage.RF_PREDICTION.value][constants.STATE] + rf_prediction_state = sim_metadata[SimulationStage.RF_PREDICTION.value][ + constants.STATE + ] batch = rf_prediction_output[constants.BATCH] # TODO: update this to account for past failures once @@ -474,11 +538,15 @@ def _handle_rf_prediction_output(self, sim_metadata: Dict, rf_prediction_output: rf_prediction_state[constants.LATEST_BATCH_TO_SUCCEED] = batch # check if the stage has completed, update stage status if so - stage_completed = OrchestrationHelper.stage_has_completed(sim_metadata, SimulationStage.RF_PREDICTION) + stage_completed = OrchestrationHelper.stage_has_completed( + sim_metadata, SimulationStage.RF_PREDICTION + ) if stage_completed: rf_prediction_state[constants.STATUS] = WorkflowStatus.FINISHED.value - if OrchestrationHelper.has_stage(sim_metadata, stage=SimulationStage.PROTOCOL_EMULATION): + if OrchestrationHelper.has_stage( + sim_metadata, stage=SimulationStage.PROTOCOL_EMULATION + ): self._start_single_protocol_emulation_job( sim_metadata, simulation_id, @@ -526,17 +594,23 @@ def _wrap_up_simulation(self, sim_metadata: Dict, simulation_id: str): sim_metadata[constants.SIMULATION_STATUS] = WorkflowStatus.FINISHED.value logger.info(f"Successfully completed simulation: {simulation_id}") - def _copy_simulation_output_to_consume_folder(self, sim_metadata: Dict, simulation_id: str): + def _copy_simulation_output_to_consume_folder( + self, sim_metadata: Dict, simulation_id: str + ): """Copy simulation output to consumable zip file Consumable zip file will allow the user to download data and system will delete afterwords. """ - logger.debug(f"Copying output from simulation: {simulation_id} to consumable zip file") + logger.debug( + f"Copying output from simulation: {simulation_id} to consumable zip file" + ) # get the output stage simulation_output_stage = OrchestrationHelper.get_output_stage(sim_metadata) - hash_val = OrchestrationHelper.get_stage_hash_val(sim_metadata, stage=simulation_output_stage) + hash_val = OrchestrationHelper.get_stage_hash_val( + sim_metadata, stage=simulation_output_stage + ) # pull number of batches to zip _, num_batches = OrchestrationHelper.get_batching_params(sim_metadata) @@ -551,11 +625,17 @@ def _copy_simulation_output_to_consume_folder(self, sim_metadata: Dict, simulati def _clean_unused_output_data(self, sim_metadata: Dict, simulation_id: str): """Clean unused output data""" - logger.info(f"Removing all outputs not used in recent simulation: {simulation_id}") + logger.info( + f"Removing all outputs not used in recent simulation: {simulation_id}" + ) - if OrchestrationHelper.has_stage(sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION): + if OrchestrationHelper.has_stage( + sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION + ): # clean UE Tracks Generation outputs - ue_tracks_generation_hash_val: Optional[str] = OrchestrationHelper.get_stage_hash_val( + ue_tracks_generation_hash_val: Optional[ + str + ] = OrchestrationHelper.get_stage_hash_val( sim_metadata, stage=SimulationStage.UE_TRACKS_GENERATION, ) @@ -564,9 +644,13 @@ def _clean_unused_output_data(self, sim_metadata: Dict, simulation_id: str): save_hash_val=ue_tracks_generation_hash_val, ) - if OrchestrationHelper.has_stage(sim_metadata, stage=SimulationStage.RF_PREDICTION): + if OrchestrationHelper.has_stage( + sim_metadata, stage=SimulationStage.RF_PREDICTION + ): # clean RF Prediction outputs - rf_prediction_hash_val: Optional[str] = OrchestrationHelper.get_stage_hash_val( + rf_prediction_hash_val: Optional[ + str + ] = OrchestrationHelper.get_stage_hash_val( sim_metadata, stage=SimulationStage.RF_PREDICTION, ) @@ -575,9 +659,13 @@ def _clean_unused_output_data(self, sim_metadata: Dict, simulation_id: str): save_hash_val=rf_prediction_hash_val, ) - if OrchestrationHelper.has_stage(sim_metadata, stage=SimulationStage.PROTOCOL_EMULATION): + if OrchestrationHelper.has_stage( + sim_metadata, stage=SimulationStage.PROTOCOL_EMULATION + ): # clean Protocol Emulation outputs - protocol_emulation_hash_val: Optional[str] = OrchestrationHelper.get_stage_hash_val( + protocol_emulation_hash_val: Optional[ + str + ] = OrchestrationHelper.get_stage_hash_val( sim_metadata, stage=SimulationStage.PROTOCOL_EMULATION, ) @@ -585,4 +673,6 @@ def _clean_unused_output_data(self, sim_metadata: Dict, simulation_id: str): stage=SimulationStage.PROTOCOL_EMULATION, save_hash_val=protocol_emulation_hash_val, ) - logger.info(f"Successfully removed all unused outputs for simulation: {simulation_id}") + logger.info( + f"Successfully removed all unused outputs for simulation: {simulation_id}" + ) diff --git a/services/orchestration/tests/test_orchestration_helper.py b/services/orchestration/tests/test_orchestration_helper.py index dce3b3e..a58ddc1 100644 --- a/services/orchestration/tests/test_orchestration_helper.py +++ b/services/orchestration/tests/test_orchestration_helper.py @@ -143,10 +143,16 @@ def test_get_rf_digital_twin_model_id(self): ) def test_has_stage(self): - self.assertTrue(OrchestrationHelper.has_stage(dummy_sim_metadata, SimulationStage.RF_PREDICTION)) + self.assertTrue( + OrchestrationHelper.has_stage( + dummy_sim_metadata, SimulationStage.RF_PREDICTION + ) + ) def test_stage_missing(self): - self.assertFalse(OrchestrationHelper.has_stage(dummy_sim_metadata, SimulationStage.START)) + self.assertFalse( + OrchestrationHelper.has_stage(dummy_sim_metadata, SimulationStage.START) + ) def test_has_hash(self): sim_metadata_hash = {"rf_prediction": {"hash_val": "val"}} @@ -217,9 +223,21 @@ def test_stage_has_completed(self): } num_ticks, num_batches = OrchestrationHelper.get_batching_params(dummy_sim_data) - self.assertTrue(OrchestrationHelper.stage_has_completed(dummy_sim_data, SimulationStage.UE_TRACKS_GENERATION)) - self.assertTrue(OrchestrationHelper.stage_has_completed(dummy_sim_data, SimulationStage.RF_PREDICTION)) - self.assertTrue(OrchestrationHelper.stage_has_completed(dummy_sim_data, SimulationStage.PROTOCOL_EMULATION)) + self.assertTrue( + OrchestrationHelper.stage_has_completed( + dummy_sim_data, SimulationStage.UE_TRACKS_GENERATION + ) + ) + self.assertTrue( + OrchestrationHelper.stage_has_completed( + dummy_sim_data, SimulationStage.RF_PREDICTION + ) + ) + self.assertTrue( + OrchestrationHelper.stage_has_completed( + dummy_sim_data, SimulationStage.PROTOCOL_EMULATION + ) + ) def test_stage_has_completed_exception(self): dummy_sim_data = { @@ -244,9 +262,21 @@ def test_stage_has_completed_exception(self): } }, } - self.assertFalse(OrchestrationHelper.stage_has_completed(dummy_sim_data, SimulationStage.UE_TRACKS_GENERATION)) - self.assertFalse(OrchestrationHelper.stage_has_completed(dummy_sim_data, SimulationStage.RF_PREDICTION)) - self.assertFalse(OrchestrationHelper.stage_has_completed(dummy_sim_data, SimulationStage.PROTOCOL_EMULATION)) + self.assertFalse( + OrchestrationHelper.stage_has_completed( + dummy_sim_data, SimulationStage.UE_TRACKS_GENERATION + ) + ) + self.assertFalse( + OrchestrationHelper.stage_has_completed( + dummy_sim_data, SimulationStage.RF_PREDICTION + ) + ) + self.assertFalse( + OrchestrationHelper.stage_has_completed( + dummy_sim_data, SimulationStage.PROTOCOL_EMULATION + ) + ) def test_update_stage_state_to_finished(self): dummy_sim_data = { @@ -259,10 +289,14 @@ def test_update_stage_state_to_finished(self): } }, "rf_prediction": {"state": {"batches_outputted": "dummy_num_batches_val"}}, - "protocol_emulation": {"state": {"batches_outputted": "dummy_num_batches_val"}}, + "protocol_emulation": { + "state": {"batches_outputted": "dummy_num_batches_val"} + }, } stage_ue_track_generation = SimulationStage.UE_TRACKS_GENERATION - OrchestrationHelper.update_stage_state_to_finished(dummy_sim_data, stage_ue_track_generation) + OrchestrationHelper.update_stage_state_to_finished( + dummy_sim_data, stage_ue_track_generation + ) self.assertEqual( dummy_sim_data[stage_ue_track_generation.value]["state"], { diff --git a/services/rf_prediction/rf_prediction_consumer.py b/services/rf_prediction/rf_prediction_consumer.py index 0532df2..f88797a 100644 --- a/services/rf_prediction/rf_prediction_consumer.py +++ b/services/rf_prediction/rf_prediction_consumer.py @@ -45,15 +45,22 @@ def consume_from_jobs(self) -> None: logger.debug("Waiting...") continue if message.error(): - logger.exception(f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}") + logger.exception( + f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}" + ) continue # Extract the (optional) key and value, and print. - logger.debug(f"Consumed message value = {message.value().decode('utf-8')}") + logger.debug( + f"Consumed message value = {message.value().decode('utf-8')}" + ) job_data = json.loads(message.value().decode("utf-8")) # ignore non-RF Prediction related jobs - if job_data[constants.KAFKA_JOB_TYPE] != constants.JOB_TYPE_RF_PREDICTION: + if ( + job_data[constants.KAFKA_JOB_TYPE] + != constants.JOB_TYPE_RF_PREDICTION + ): continue # execute RF prediction @@ -62,7 +69,9 @@ def consume_from_jobs(self) -> None: logger.info("Successfully ran RF Prediction on model") except Exception as e: # TODO: add output event failure handling here - logger.exception(f"Exception occurred while handling RF Prediction job: {job_data}\n{e}") + logger.exception( + f"Exception occurred while handling RF Prediction job: {job_data}\n{e}" + ) except KeyboardInterrupt: pass finally: diff --git a/services/rf_prediction/rf_prediction_driver.py b/services/rf_prediction/rf_prediction_driver.py index e23da47..62f7679 100644 --- a/services/rf_prediction/rf_prediction_driver.py +++ b/services/rf_prediction/rf_prediction_driver.py @@ -75,7 +75,7 @@ def handle_rf_prediction_job(self, job_data): # pull the model model_id, model_file_path = RFPredictionHelper.get_model_parameters(job_data) - + job_id = job_data[constants.JOB_ID] logger.info(f"Running RF prediction using model: {model_id}") @@ -102,11 +102,17 @@ def handle_rf_prediction_job(self, job_data): ) # load model map - bayesian_digital_twin_map = BayesianDigitalTwin.load_model_map_from_pickle(model_file_path=model_file_path) - logger.debug(f"Loaded bayesian digital twin model {model_id} from '{model_file_path}'") + bayesian_digital_twin_map = BayesianDigitalTwin.load_model_map_from_pickle( + model_file_path=model_file_path + ) + logger.debug( + f"Loaded bayesian digital twin model {model_id} from '{model_file_path}'" + ) # run per-cell prediction - prediction_output_map = self._run_inference_per_cell(cell_id_ue_data_map, bayesian_digital_twin_map) + prediction_output_map = self._run_inference_per_cell( + cell_id_ue_data_map, bayesian_digital_twin_map + ) # attach per-cell rxpower_dbm to ue_data_df rx_powers = [] @@ -116,7 +122,9 @@ def handle_rf_prediction_job(self, job_data): # get the expected RF Prediction output data # TODO: Reorder to first pull then insert rx_powers - ue_data_df.insert(loc=len(ue_data_df.columns), column=constants.RXPOWER_DBM, value=rx_powers) + ue_data_df.insert( + loc=len(ue_data_df.columns), column=constants.RXPOWER_DBM, value=rx_powers + ) rf_prediction_output = ue_data_df.loc[ :, [ @@ -129,7 +137,9 @@ def handle_rf_prediction_job(self, job_data): # if mock_ue_id provided, append to output if constants.MOCK_UE_ID in ue_data_df: - rf_prediction_output[constants.MOCK_UE_ID] = ue_data_df[constants.MOCK_UE_ID] + rf_prediction_output[constants.MOCK_UE_ID] = ue_data_df[ + constants.MOCK_UE_ID + ] # if tick provided, append to output if constants.TICK in ue_data_df: @@ -151,7 +161,9 @@ def handle_rf_prediction_job(self, job_data): constants.BATCH: batch, constants.STATUS: OutputStatus.SUCCESS.value, } - produce_object_to_kafka_topic(self.producer, topic=constants.OUTPUTS, value=output_event) + produce_object_to_kafka_topic( + self.producer, topic=constants.OUTPUTS, value=output_event + ) logger.info(f"Produced successful output event to topic: {output_event}") def _preprocess_ue_data( @@ -176,7 +188,9 @@ def _preprocess_ue_data( # perform cross replication if required if cross_replications_required: - logger.info("No cell_id column found in UE data, running cross replication...") + logger.info( + "No cell_id column found in UE data, running cross replication..." + ) cell_ids = pd.DataFrame(config_df[constants.CELL_ID]) ue_data_df = cross_replicate(ue_data_df, cell_ids) @@ -184,7 +198,9 @@ def _preprocess_ue_data( logger.info("Finished running cross replication!") # run Bayesian digital twin preprocessing - cell_id_ue_data_map: Dict[str, pd.DataFrame] = BayesianDigitalTwin.preprocess_ue_prediction_data( + cell_id_ue_data_map: Dict[ + str, pd.DataFrame + ] = BayesianDigitalTwin.preprocess_ue_prediction_data( ue_data_df=ue_data_df, config_df=config_df, topology_df=topology_df, @@ -205,9 +221,9 @@ def _run_inference_per_cell( for cell_id, ue_prediction_data in cell_id_ue_data_map.items(): logger.info(f"Running inference for cell_id: {cell_id}...") # run prediction - pred_means, pred_std = bayesian_digital_twin_map[cell_id].predict_distributed_gpmodel( - prediction_dfs=[ue_prediction_data] - ) + pred_means, pred_std = bayesian_digital_twin_map[ + cell_id + ].predict_distributed_gpmodel(prediction_dfs=[ue_prediction_data]) # store to output map prediction_output_map[cell_id] = (pred_means, pred_std, ue_prediction_data) diff --git a/services/training/training_consumer.py b/services/training/training_consumer.py index f9318af..9ccd360 100644 --- a/services/training/training_consumer.py +++ b/services/training/training_consumer.py @@ -45,11 +45,15 @@ def consume_from_jobs(self): logger.debug("Waiting...") continue if message.error(): - logger.exception(f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}") + logger.exception( + f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}" + ) continue # Extract the (optional) key and value, and print. - logger.debug(f"Consumed message value = {message.value().decode('utf-8')}") + logger.debug( + f"Consumed message value = {message.value().decode('utf-8')}" + ) job_data = json.loads(message.value().decode("utf-8")) # ignore non-training related jobs @@ -59,9 +63,13 @@ def consume_from_jobs(self): # execute training try: self.training_driver.handle_training_job(job_data) - logger.info(f"Successfully trained model {job_data[constants.MODEL_ID]}") + logger.info( + f"Successfully trained model {job_data[constants.MODEL_ID]}" + ) except Exception as e: - logger.exception(f"Exception occurred while handling training job: {job_data}\n{e}") + logger.exception( + f"Exception occurred while handling training job: {job_data}\n{e}" + ) except KeyboardInterrupt: pass finally: diff --git a/services/training/training_driver.py b/services/training/training_driver.py index 6705051..26ca30a 100644 --- a/services/training/training_driver.py +++ b/services/training/training_driver.py @@ -57,12 +57,16 @@ def handle_training_job(self, job_data: Dict): ue_training_data_df = pd.read_csv(ue_training_data_file) topology_df = pd.read_csv(topology_file) except Exception as e: - logger.exception(f"Exception occurred while loading digital twin training data: {e}") + logger.exception( + f"Exception occurred while loading digital twin training data: {e}" + ) raise Exception # Preprocess training data logger.info("Preprocessing training data...") - cell_id_ue_data_map = BayesianDigitalTwin.preprocess_ue_training_data(ue_training_data_df, topology_df) + cell_id_ue_data_map = BayesianDigitalTwin.preprocess_ue_training_data( + ue_training_data_df, topology_df + ) logger.info("Finished preprocessing training data...") logger.info("Starting model training...") @@ -101,9 +105,7 @@ def handle_training_job(self, job_data: Dict): # prime the model cache by calling a mock prediction on it # using the first set of training data # so that it is ready for further updates or operations - model_map[cell_id].predict_distributed_gpmodel( - [training_data.head(1)] - ) + model_map[cell_id].predict_distributed_gpmodel([training_data.head(1)]) # save the serialized model map object to file BayesianDigitalTwin.save_model_map_to_pickle( @@ -116,7 +118,9 @@ def handle_training_job(self, job_data: Dict): model_metadata[constants.STATUS] = ModelStatus.TRAINED.value model_metadata[constants.JOB_ID] = job_id - model_metadata[constants.JOB_FINISHED_DATETIME] = datetime.datetime.now(datetime.timezone.utc).isoformat() + model_metadata[constants.JOB_FINISHED_DATETIME] = datetime.datetime.now( + datetime.timezone.utc + ).isoformat() RADPFileSystemHelper.save_model_metadata( model_id=model_id, @@ -147,7 +151,7 @@ def _train_model( training_params: Dict, ) -> BayesianDigitalTwin: """Train a model per each cell of data""" - + # this class init method fully prepares the model to be trained # but stops just short of actually calling train() on it model = BayesianDigitalTwin( diff --git a/services/ue_tracks_generation/main.py b/services/ue_tracks_generation/main.py index 241a165..4312060 100644 --- a/services/ue_tracks_generation/main.py +++ b/services/ue_tracks_generation/main.py @@ -6,7 +6,9 @@ import signal import sys -from ue_tracks_generation.ue_tracks_generation_consumer import UETracksGenerationConsumer +from ue_tracks_generation.ue_tracks_generation_consumer import ( + UETracksGenerationConsumer, +) # define a sigterm handler to allow docker to gracefully exit diff --git a/services/ue_tracks_generation/ue_tracks_generation_consumer.py b/services/ue_tracks_generation/ue_tracks_generation_consumer.py index 1d64726..8e0e247 100644 --- a/services/ue_tracks_generation/ue_tracks_generation_consumer.py +++ b/services/ue_tracks_generation/ue_tracks_generation_consumer.py @@ -27,7 +27,9 @@ def __init__(self): self.ue_tracks_generation_driver = UETracksGenerationDriver() # subscribe to topics - safe_subscribe(consumer=self.consumer, topics=UE_TRACKS_GENERATION_CONSUMER_TOPICS) + safe_subscribe( + consumer=self.consumer, topics=UE_TRACKS_GENERATION_CONSUMER_TOPICS + ) logger.info(f"Subscribed to topics: {UE_TRACKS_GENERATION_CONSUMER_TOPICS}") def consume_from_jobs(self) -> None: @@ -45,23 +47,34 @@ def consume_from_jobs(self) -> None: logger.debug("Waiting...") continue if message.error(): - logger.exception(f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}") + logger.exception( + f"Error consuming from {constants.KAFKA_JOBS_TOPIC_NAME} topic: {message.error()}" + ) continue # Extract the (optional) key and value, and print. - logger.debug(f"Consumed message value = {message.value().decode('utf-8')}") + logger.debug( + f"Consumed message value = {message.value().decode('utf-8')}" + ) job_data = json.loads(message.value().decode("utf-8")) # ignore non-ue_tracks_generation related jobs - if job_data[constants.KAFKA_JOB_TYPE] != constants.JOB_TYPE_UE_TRACKS_GENERATION: + if ( + job_data[constants.KAFKA_JOB_TYPE] + != constants.JOB_TYPE_UE_TRACKS_GENERATION + ): continue # execute ue_tracks_generation try: - self.ue_tracks_generation_driver.handle_ue_tracks_generation_job(job_data) + self.ue_tracks_generation_driver.handle_ue_tracks_generation_job( + job_data + ) logger.info("Successfully executed UE Tracks Generation job") except Exception as e: - logger.exception(f"Exception occurred while handling ue_tracks_generation job: {job_data}\n{e}") + logger.exception( + f"Exception occurred while handling ue_tracks_generation job: {job_data}\n{e}" + ) except KeyboardInterrupt: pass finally: diff --git a/services/ue_tracks_generation/ue_tracks_generation_driver.py b/services/ue_tracks_generation/ue_tracks_generation_driver.py index 0f6fbef..8094da3 100644 --- a/services/ue_tracks_generation/ue_tracks_generation_driver.py +++ b/services/ue_tracks_generation/ue_tracks_generation_driver.py @@ -108,11 +108,19 @@ def handle_ue_tracks_generation_job(self, job_data=None): logger.info(f"Handling UE Tracks generation job: {job_data}") # Extract all the required information from the job_data in order to generate UE tracks - ue_tracks_generation_params = UETracksGenerationHelper.get_ue_tracks_generation_parameters(job_data) + ue_tracks_generation_params = ( + UETracksGenerationHelper.get_ue_tracks_generation_parameters(job_data) + ) - simulation_time_interval = UETracksGenerationHelper.get_simulation_time_interval(ue_tracks_generation_params) + simulation_time_interval = ( + UETracksGenerationHelper.get_simulation_time_interval( + ue_tracks_generation_params + ) + ) num_ticks = UETracksGenerationHelper.get_num_ticks(ue_tracks_generation_params) - num_batches = UETracksGenerationHelper.get_num_batches(ue_tracks_generation_params) + num_batches = UETracksGenerationHelper.get_num_batches( + ue_tracks_generation_params + ) # Get the total number of UEs from the UE class distribution and add them up ( @@ -120,7 +128,9 @@ def handle_ue_tracks_generation_job(self, job_data=None): pedestrian_count, cyclist_count, car_count, - ) = UETracksGenerationHelper.get_ue_class_distribution_count(ue_tracks_generation_params) + ) = UETracksGenerationHelper.get_ue_class_distribution_count( + ue_tracks_generation_params + ) num_UEs = stationary_count + pedestrian_count + cyclist_count + car_count @@ -188,11 +198,19 @@ def handle_ue_tracks_generation_job(self, job_data=None): ) = UETracksGenerationHelper.get_lat_lon_boundaries(ue_tracks_generation_params) # Gauss Markov params - alpha = UETracksGenerationHelper.get_gauss_markov_alpha(ue_tracks_generation_params) - variance = UETracksGenerationHelper.get_gauss_markov_variance(ue_tracks_generation_params) - rng_seed = UETracksGenerationHelper.get_gauss_markov_rng_seed(ue_tracks_generation_params) + alpha = UETracksGenerationHelper.get_gauss_markov_alpha( + ue_tracks_generation_params + ) + variance = UETracksGenerationHelper.get_gauss_markov_variance( + ue_tracks_generation_params + ) + rng_seed = UETracksGenerationHelper.get_gauss_markov_rng_seed( + ue_tracks_generation_params + ) - lon_x_dims, lon_y_dims = UETracksGenerationHelper.get_gauss_markov_xy_dims(ue_tracks_generation_params) + lon_x_dims, lon_y_dims = UETracksGenerationHelper.get_gauss_markov_xy_dims( + ue_tracks_generation_params + ) # Use the above parameters extracted from the job data to generate mobility # Get each batch of mobility data in form of DataFrames @@ -200,7 +218,9 @@ def handle_ue_tracks_generation_job(self, job_data=None): simulation_id = UETracksGenerationHelper.get_simulation_id(job_data) current_batch = 1 - for ue_tracks_generation_current_batch_df in UETracksGenerator.generate_as_lon_lat_points( + for ( + ue_tracks_generation_current_batch_df + ) in UETracksGenerator.generate_as_lon_lat_points( rng_seed=rng_seed, lon_x_dims=lon_x_dims, lon_y_dims=lon_y_dims, @@ -218,12 +238,20 @@ def handle_ue_tracks_generation_job(self, job_data=None): mobility_class_velocity_variances=mobility_class_velocity_variances, ): # save output to file with format {output_file_prefix}-{batch}.fea - output_file_prefix = UETracksGenerationHelper.get_output_file_prefix(job_data) - output_file_name = f"{output_file_prefix}-{current_batch}.{constants.DF_FILE_EXTENSION}" - output_file_path = os.path.join(constants.UE_TRACK_GENERATION_OUTPUTS_FOLDER, output_file_name) + output_file_prefix = UETracksGenerationHelper.get_output_file_prefix( + job_data + ) + output_file_name = ( + f"{output_file_prefix}-{current_batch}.{constants.DF_FILE_EXTENSION}" + ) + output_file_path = os.path.join( + constants.UE_TRACK_GENERATION_OUTPUTS_FOLDER, output_file_name + ) write_feather_df(output_file_path, ue_tracks_generation_current_batch_df) - logger.info(f"Saved UE Tracks batch {current_batch} output DF to {output_file_path}") + logger.info( + f"Saved UE Tracks batch {current_batch} output DF to {output_file_path}" + ) # Once each batch has been processed and written to the output file, we can indicate that the job # has done successfully and produce output event to outputs topic diff --git a/tests/run_component_tests.py b/tests/run_component_tests.py index 3b3c696..945c008 100644 --- a/tests/run_component_tests.py +++ b/tests/run_component_tests.py @@ -34,7 +34,9 @@ def get_top_level(relative_path): def get_package_roots(top_level_path): """Helper method to get root packages from relative path""" - root_paths = [os.path.join(top_level_path, path) for path in os.listdir(top_level_path)] + root_paths = [ + os.path.join(top_level_path, path) for path in os.listdir(top_level_path) + ] return [path for path in root_paths if os.path.isdir(path)] @@ -46,7 +48,9 @@ def run_tests(source_paths) -> Tuple[List[unittest.TestResult], int]: total_tests = 0 for path in source_paths: - test_suite = loader.discover(start_dir=path, top_level_dir=path, pattern="test*.py") + test_suite = loader.discover( + start_dir=path, top_level_dir=path, pattern="test*.py" + ) total_tests += test_suite.countTestCases() testRunner = unittest.runner.TextTestRunner() @@ -74,13 +78,17 @@ def run_tests(source_paths) -> Tuple[List[unittest.TestResult], int]: cov.start() # run tests - test_results, total_tests = run_tests([radp_top_level_path, services_top_level_path]) + test_results, total_tests = run_tests( + [radp_top_level_path, services_top_level_path] + ) # stop coverage tracking cov.stop() else: # run tests - test_results, total_tests = run_tests([radp_top_level_path, services_top_level_path]) + test_results, total_tests = run_tests( + [radp_top_level_path, services_top_level_path] + ) success = True failed: List[Tuple[unittest.TestResult, str]] = [] @@ -126,7 +134,9 @@ def run_tests(source_paths) -> Tuple[List[unittest.TestResult], int]: logger.info(f"{'ERRORS:' : <20}{error_count : >36}") logger.info("--------------------------------------------------------") -message = "SUCCESS" if success else "TESTS FAILED: Check test reports above to see failure(s)" +message = ( + "SUCCESS" if success else "TESTS FAILED: Check test reports above to see failure(s)" +) logger.info(message) logger.info("--------------------------------------------------------") diff --git a/tests/run_end_to_end_tests.py b/tests/run_end_to_end_tests.py index a4171ce..6ebf199 100644 --- a/tests/run_end_to_end_tests.py +++ b/tests/run_end_to_end_tests.py @@ -9,7 +9,9 @@ from datetime import datetime from happy_case_tests.happy_rf_prediction import happy_case__rf_prediction -from happy_case_tests.happy_ue_track_gen_rf_prediction import happy_case__ue_track_gen_rf_prediction +from happy_case_tests.happy_ue_track_gen_rf_prediction import ( + happy_case__ue_track_gen_rf_prediction, +) # Create the Logger logger = logging.getLogger(__name__) @@ -64,19 +66,25 @@ logger.info("") logger.info(f"{'Result:' : <20}{'SUCCESS' if test_passed else 'FAILED' : >36}") - logger.info(f"{'Test duration:' : <20}{str(datetime.now() - test_start_time) : >36}") + logger.info( + f"{'Test duration:' : <20}{str(datetime.now() - test_start_time) : >36}" + ) # report results logger.info("--------------------------------------------------------") logger.info("--------------------------------------------------------") -logger.info(f"{'TEST RESULT SUMMARY:' : <24}{'SUCCESS' if all_passed else 'FAILURE' : >32}") +logger.info( + f"{'TEST RESULT SUMMARY:' : <24}{'SUCCESS' if all_passed else 'FAILURE' : >32}" +) logger.info(f"{'TESTS RUN:' : <20}{run : >36}") logger.info(f"{'TESTS PASSED:' : <20}{passed : >36}") logger.info(f"{'TESTS FAILED:' : <20}{failed : >36}") logger.info(f"{'TESTS DURATION:' : <20}{str(datetime.now() - start_time) : >36}") logger.info("--------------------------------------------------------") -message = "SUCCESS" if all_passed else "TESTS FAILED: Check logs above to see failure(s)" +message = ( + "SUCCESS" if all_passed else "TESTS FAILED: Check logs above to see failure(s)" +) logger.info(message) logger.info("--------------------------------------------------------")