From 86dce736f121e91f432deb6ef3c932d2173dc638 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Wed, 23 Nov 2022 11:18:23 -0500 Subject: [PATCH 01/50] chore: bump mypy to version 0.991 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41f09c72c..a8b82cf63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - "--extend-ignore=E203" - "--per-file-ignores=__init__.py:F401" - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.971" + rev: "v0.991" hooks: - id: mypy additional_dependencies: [types-requests] From 5f4805f2a3b0c7ddfe22f2efd7be032da19b7379 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Wed, 23 Nov 2022 11:18:41 -0500 Subject: [PATCH 02/50] chore: add explicit optional to type annotations --- autora/theorist/darts/architect.py | 6 ++-- autora/theorist/darts/model_search.py | 6 ++-- autora/theorist/darts/plot_utils.py | 45 ++++++++++++++------------- autora/theorist/darts/utils.py | 18 +++++------ autora/theorist/darts/visualize.py | 11 ++++--- 5 files changed, 46 insertions(+), 40 deletions(-) diff --git a/autora/theorist/darts/architect.py b/autora/theorist/darts/architect.py index 63bc09f01..c7d723e29 100755 --- a/autora/theorist/darts/architect.py +++ b/autora/theorist/darts/architect.py @@ -1,3 +1,5 @@ +from typing import Optional + import numpy as np import torch import torch.nn.functional as F @@ -150,8 +152,8 @@ def step( target_valid: torch.Tensor, network_optimizer: torch.optim.Optimizer, unrolled: bool, - input_train: torch.Tensor = None, - target_train: torch.Tensor = None, + input_train: Optional[torch.Tensor] = None, + target_train: Optional[torch.Tensor] = None, eta: float = 1, ): """ diff --git a/autora/theorist/darts/model_search.py b/autora/theorist/darts/model_search.py index 490675912..bebc2fa8f 100755 --- a/autora/theorist/darts/model_search.py +++ b/autora/theorist/darts/model_search.py @@ -1,7 +1,7 @@ import random import warnings from enum import Enum -from typing import Callable, List, Literal, Sequence, Tuple +from typing import Callable, List, Literal, Optional, Sequence, Tuple import numpy as np import torch @@ -400,7 +400,9 @@ def arch_parameters(self) -> List: return self._arch_parameters # fixes architecture - def fix_architecture(self, switch: bool, new_weights: torch.Tensor = None): + def fix_architecture( + self, switch: bool, new_weights: Optional[torch.Tensor] = None + ): """ Freezes or unfreezes the architecture weights. diff --git a/autora/theorist/darts/plot_utils.py b/autora/theorist/darts/plot_utils.py index 6524393a6..d256bf412 100755 --- a/autora/theorist/darts/plot_utils.py +++ b/autora/theorist/darts/plot_utils.py @@ -1,5 +1,6 @@ import os import typing +from typing import Optional import imageio import matplotlib @@ -33,9 +34,9 @@ def generate_darts_summary_figures( y_limit: typing.List[float], best_model_name: str, figure_size: typing.Tuple[int, int], - y_reference: typing.List[float] = None, + y_reference: Optional[typing.List[float]] = None, y_reference_label: str = "", - arch_samp_filter: str = None, + arch_samp_filter: Optional[str] = None, ): """ Generates a summary figure for a given DARTS study. @@ -118,22 +119,22 @@ def plot_darts_summary( y_label: str = "", x1_label: str = "", x2_label: str = "", - y_sem_name: str = None, + y_sem_name: Optional[str] = None, metric: str = "min", - y_reference: typing.List[float] = None, + y_reference: Optional[typing.List[float]] = None, y_reference_label: str = "", - figure_dimensions: typing.Tuple[int, int] = None, + figure_dimensions: Optional[typing.Tuple[int, int]] = None, title: str = "", legend_loc: int = 0, legend_font_size: int = 8, axis_font_size: int = 10, title_font_size: int = 10, show_legend: bool = True, - y_limit: typing.List[float] = None, - x_limit: typing.List[float] = None, - theorist_filter: str = None, - arch_samp_filter: str = None, - best_model_name: str = None, + y_limit: Optional[typing.List[float]] = None, + x_limit: Optional[typing.List[float]] = None, + theorist_filter: Optional[str] = None, + arch_samp_filter: Optional[str] = None, + best_model_name: Optional[str] = None, save: bool = False, figure_name: str = "figure", ): @@ -996,18 +997,18 @@ def __init__( def update( self, - train_error: np.array = None, - valid_error: np.array = None, - weights: np.array = None, - BIC: np.array = None, - AIC: np.array = None, - model_graph: str = None, - range_input1: np.array = None, - range_input2: np.array = None, - range_target: np.array = None, - range_prediction: np.array = None, - target: np.array = None, - prediction: np.array = None, + train_error: Optional[np.array] = None, + valid_error: Optional[np.array] = None, + weights: Optional[np.array] = None, + BIC: Optional[np.array] = None, + AIC: Optional[np.array] = None, + model_graph: Optional[str] = None, + range_input1: Optional[np.array] = None, + range_input2: Optional[np.array] = None, + range_target: Optional[np.array] = None, + range_prediction: Optional[np.array] = None, + target: Optional[np.array] = None, + prediction: Optional[np.array] = None, ): """ Update the debug plot with new data. diff --git a/autora/theorist/darts/utils.py b/autora/theorist/darts/utils.py index 7fa8b467f..79d6affbc 100755 --- a/autora/theorist/darts/utils.py +++ b/autora/theorist/darts/utils.py @@ -2,7 +2,7 @@ import glob import os import shutil -from typing import Callable, List, Tuple +from typing import Callable, List, Optional, Tuple import numpy as np import torch @@ -14,11 +14,11 @@ def create_output_file_name( file_prefix: str, - log_version: int = None, - weight_decay: float = None, - k: int = None, - seed: int = None, - theorist: str = None, + log_version: Optional[int] = None, + weight_decay: Optional[float] = None, + k: Optional[int] = None, + seed: Optional[int] = None, + theorist: Optional[str] = None, ) -> str: """ Creates a file name for the output file of a theorist study. @@ -278,7 +278,7 @@ def count_parameters_in_MB(model: Network) -> int: ) -def save(model: torch.nn.Module, model_path: str, exp_folder: str = None): +def save(model: torch.nn.Module, model_path: str, exp_folder: Optional[str] = None): """ Saves a model to a file. @@ -303,9 +303,9 @@ def load(model: torch.nn.Module, model_path: str): def create_exp_dir( path: str, - scripts_to_save: List = None, + scripts_to_save: Optional[List] = None, parent_folder: str = "exps", - results_folder: str = None, + results_folder: Optional[str] = None, ): """ Creates an experiment directory and saves all necessary scripts and files. diff --git a/autora/theorist/darts/visualize.py b/autora/theorist/darts/visualize.py index ac6648211..bf3055332 100755 --- a/autora/theorist/darts/visualize.py +++ b/autora/theorist/darts/visualize.py @@ -1,5 +1,6 @@ import logging import typing +from typing import Optional from graphviz import Digraph @@ -12,12 +13,12 @@ def plot( genotype: Genotype, filename: str, file_format: str = "pdf", - view_file: bool = None, + view_file: Optional[bool] = None, full_label: bool = False, param_list: typing.Tuple = (), input_labels: typing.Tuple = (), - out_dim: int = None, - out_fnc: str = None, + out_dim: Optional[int] = None, + out_fnc: Optional[str] = None, ): """ Generates a graphviz plot for a DARTS model based on the genotype of the model. @@ -58,8 +59,8 @@ def darts_model_plot( full_label: bool = False, param_list: typing.Sequence = (), input_labels: typing.Sequence = (), - out_dim: int = None, - out_fnc: str = None, + out_dim: Optional[int] = None, + out_fnc: Optional[str] = None, decimals_to_display: int = 2, ) -> Digraph: """ From 65ae0ce656ab15cc126c0b85fcab35064f7fbaa6 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Wed, 23 Nov 2022 11:30:58 -0500 Subject: [PATCH 03/50] chore: bump pre-commit black version to 22.10.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41f09c72c..03cf8099e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/ambv/black - rev: 22.8.0 + rev: 22.10.0 hooks: - id: black - repo: https://github.com/pycqa/isort From 5648836151f6edba7d5a6f3fd1d34d613a026412 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Fri, 2 Dec 2022 16:34:57 -0500 Subject: [PATCH 04/50] feat: add doctests for train_test_split --- autora/experimentalist/filter.py | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index ff2bc6873..24d649367 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -1,2 +1,67 @@ +import numpy as np + + def weber_filter(values): return filter(lambda s: s[0] <= s[1], values) + + +def train_test_filter(seed=180, train_p=0.5): + """ + A pipeline filter which pseudorandomly assigns values from the input into "train" or "test" + groups. + + Examples: + We can create complementary train and test filters using the function: + >>> train_filter, test_filter = train_test_filter(train_p=0.6, seed=180) + + The train filter generates a sequence of 60% of the input list. + >>> list(train_filter(range(20))) + [0, 2, 3, 4, 5, 6, 9, 10, 11, 12, 15, 16, 17, 18, 19] + + When we run the test_filter, it fills in the gaps. + >>> list(test_filter(range(20))) + [1, 7, 8, 13, 14] + + We can continue to generate new values for as long as we like using the same filter and the + continuation of the input range: + >>> list(train_filter(range(20, 40))) + [20, 22, 23, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 38, 39] + + ... and some more. + >>> list(train_filter(range(40, 50))) + [41, 42, 44, 45, 46, 49] + + The test_filter fills in the gaps again. + >>> list(test_filter(range(20, 30))) + [21, 24, 25, 26] + + If you rerun the *same* test_filter on a fresh range, then the results will be different + to the first time around: + >>> list(test_filter(range(20))) + [5, 10, 13, 17, 18] + + ... but if you regenerate the test_filter, it'll reproduce the original sequence + >>> _, test_filter_regenerated = train_test_filter(train_p=0.6, seed=180) + >>> list(test_filter_regenerated(range(20))) + [1, 7, 8, 13, 14] + + """ + + test_p = 1 - train_p + + def _train_test_stream(): + rng = np.random.default_rng(seed) + while True: + yield rng.choice(["train", "test"], p=(train_p, test_p)).item() + + def _factory(allow): + _stream = _train_test_stream() + + def _generator(values): + for v, train_test in zip(values, _stream): + if train_test == allow: + yield v + + return _generator + + return _factory("train"), _factory("test") From 706e57c3a56ecffc9c5273b5d3ffb518950d0b8b Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Fri, 2 Dec 2022 16:38:22 -0500 Subject: [PATCH 05/50] refactor: use train/test enum in train_test_filter --- autora/experimentalist/filter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index 24d649367..165e098f1 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -1,3 +1,5 @@ +from enum import Enum + import numpy as np @@ -49,10 +51,12 @@ def train_test_filter(seed=180, train_p=0.5): test_p = 1 - train_p + _TrainTest = Enum("_TrainTest", ["train", "test"]) + def _train_test_stream(): rng = np.random.default_rng(seed) while True: - yield rng.choice(["train", "test"], p=(train_p, test_p)).item() + yield rng.choice([_TrainTest.train, _TrainTest.test], p=(train_p, test_p)) def _factory(allow): _stream = _train_test_stream() @@ -64,4 +68,4 @@ def _generator(values): return _generator - return _factory("train"), _factory("test") + return _factory(_TrainTest.train), _factory(_TrainTest.test) From 19eb78c164620a240b6152d3fd4ab40f2c75fdb4 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Fri, 2 Dec 2022 16:58:32 -0500 Subject: [PATCH 06/50] docs: add doctests for infinite train-test-filter --- autora/experimentalist/filter.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index 165e098f1..7c106a2a9 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -47,6 +47,33 @@ def train_test_filter(seed=180, train_p=0.5): >>> list(test_filter_regenerated(range(20))) [1, 7, 8, 13, 14] + + It also works on tuple-valued lists: + >>> from itertools import product + >>> train_filter_tuple, test_filter_tuple = train_test_filter(train_p=0.3, seed=42) + >>> list(test_filter_tuple(product(["a", "b"], [1, 2, 3]))) + [('a', 1), ('a', 2), ('a', 3), ('b', 1), ('b', 3)] + + >>> list(train_filter_tuple(product(["a","b"], [1,2,3]))) + [('b', 2)] + + >>> from itertools import count, takewhile + >>> train_filter_unbounded, test_filter_unbounded = train_test_filter(train_p=0.5, seed=21) + + >>> list(takewhile(lambda s: s < 90, count(79))) + [79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + + >>> train_pool = train_filter_unbounded(count(79)) + >>> list(takewhile(lambda s: s < 90, train_pool)) + [82, 85, 86, 89] + + >>> test_pool = test_filter_unbounded(count(79)) + >>> list(takewhile(lambda s: s < 90, test_pool)) + [79, 80, 81, 83, 84, 87, 88] + + >>> list(takewhile(lambda s: s < 110, test_pool)) + [91, 93, 94, 97, 100, 105, 106, 109] + """ test_p = 1 - train_p From 9177075f6c608fa44af817cebb44bff840f3964c Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Sat, 3 Dec 2022 19:46:24 -0500 Subject: [PATCH 07/50] completed popper net sampler and pooler --- autora/experimentalist/sampler/poppernet.py | 388 ++++++++++++++++++++ tests/test_poppernet_sampler.py | 171 +++++++++ 2 files changed, 559 insertions(+) create mode 100644 autora/experimentalist/sampler/poppernet.py create mode 100644 tests/test_poppernet_sampler.py diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py new file mode 100644 index 000000000..dafcfba0f --- /dev/null +++ b/autora/experimentalist/sampler/poppernet.py @@ -0,0 +1,388 @@ +from typing import Iterable, Tuple, cast + +import numpy as np +import torch +from torch import nn +from torch.autograd import Variable + +from autora.variable import ValueType, VariableCollection + + +def poppernet_pooler( + model, + X_train: np.ndarray, + Y_train: np.ndarray, + meta_data: VariableCollection, + num_samples: int = 100, + training_epochs: int = 1000, + optimization_epochs: int = 1000, + training_lr: float = 1e-3, + optimization_lr: float = 1e-3, + mse_scale: float = 1, + limit_offset: float = 10**-10, + limit_repulsion: float = 0, + verbose: bool = False, +): + """ + A pooler that generates samples for independent variables with the objective to maximize the + (approximated) loss of the model. The samples are generated by first training a neural network + to approximate the loss of a model for all patterns in the training data. Once trained, the + network is then inverted to generate samples that maximize the approximated loss of the model. + + Note: If the pooler returns samples that are close to the boundaries of the variable space, + then it is advisable to increase the limit_repulsion parameter (e.g., to 0.000001). + + Args: + model: Scikit-learn model, could be either a classification or regression model + X_train: data that the model was trained on + Y_train: labels that the model was trained on + meta_data: Meta-data about the dependent and independent variables + num_samples: number of samples to return + training_epochs: number of epochs to train the popper network for approximating the + error fo the model + optimization_epochs: number of epochs to optimize the samples based on the trained + popper network + training_lr: learning rate for training the popper network + optimization_lr: learning rate for optimizing the samples + mse_scale: scale factor for the MSE loss + limit_offset: a limited offset to prevent the samples from being too close to the value + boundaries + limit_repulsion: a limited repulsion to prevent the samples from being too close to the + allowed value boundaries + verbose: print out the prediction of the popper network as well as its training loss + + Returns: Sampled pool + + """ + + # format input + + X_train = np.array(X_train) + if len(X_train.shape) == 1: + X_train = X_train.reshape(-1, 1) + + X = np.empty([num_samples, X_train.shape[1]]) + + Y_train = np.array(Y_train) + if len(Y_train.shape) == 1: + Y_train = Y_train.reshape(-1, 1) + + if meta_data.dependent_variables[0].type == ValueType.CLASS: + # find all unique values in Y_train + num_classes = len(np.unique(Y_train)) + Y_train = class_to_onehot(Y_train, n_classes=num_classes) + + X_train_tensor = torch.from_numpy(X_train).float() + + # create list of IV limits + IVs = meta_data.independent_variables + IV_limit_list = list() + for IV in IVs: + if hasattr(IV, "value_range"): + value_range = cast(Tuple, IV.value_range) + lower_bound = value_range[0] + upper_bound = value_range[1] + IV_limit_list.append(([lower_bound, upper_bound])) + + # get dimensions of input and output + n_input = len(meta_data.independent_variables) + n_output = len(meta_data.dependent_variables) + + # get input pattern for popper net + popper_input = Variable(torch.from_numpy(X_train), requires_grad=False).float() + + # get target pattern for popper net + model_predict = getattr(model, "predict_proba", None) + if callable(model_predict) is False: + model_predict = getattr(model, "predict", None) + + if callable(model_predict) is False or model_predict is None: + raise Exception("Model must have `predict` or `predict_proba` method.") + + model_prediction = model_predict(X_train) + + criterion = nn.MSELoss() + model_loss = (model_prediction - Y_train) ** 2 * mse_scale + model_loss = np.mean(model_loss, axis=1) + model_loss = torch.from_numpy(model_loss).float() + popper_target = Variable(model_loss, requires_grad=False) + + # create the network + popper_net = PopperNet(n_input, n_output) + + # reformat input in case it is 1D + if len(popper_input.shape) == 1: + popper_input = popper_input.flatten() + popper_input = popper_input.reshape(-1, 1) + + # define the optimizer + popper_optimizer = torch.optim.Adam(popper_net.parameters(), lr=training_lr) + + # train the network + losses = [] + for epoch in range(training_epochs): # train for 10 epochs + popper_prediction = popper_net(popper_input) + loss = criterion(popper_prediction, popper_target.reshape(-1, 1)) + popper_optimizer.zero_grad() + loss.backward() + popper_optimizer.step() + losses.append(loss.item()) + + if verbose: + print("Finished training Popper Network...") + + import matplotlib.pyplot as plt + + if len(popper_input.shape) > 0: + plot_input = popper_input[:, 0] + plt.scatter(plot_input, popper_target.detach().numpy(), label="target") + plt.scatter( + plot_input, popper_prediction.detach().numpy(), label="prediction" + ) + else: + plot_input = popper_input + plt.plot(plot_input, popper_target.detach().numpy(), label="target") + plt.plot(plot_input, popper_prediction.detach().numpy(), label="prediction") + + plt.xlabel("x") + plt.ylabel("y") + plt.legend() + plt.show() + + plt.plot(losses) + plt.xlabel("epoch") + plt.ylabel("loss") + plt.show() + + # now that the popper network is trained we can sample new data points + # to sample data points we need to provide the popper network with an initial condition + # we will sample those initial conditions proportional to the loss of the current model + + # feed avarage model losses through softmax + # model_loss_avg= torch.from_numpy(np.mean(model_loss.detach().numpy(), axis=1)).float() + softmax_func = torch.nn.Softmax(dim=0) + probabilities = softmax_func(model_loss) + # sample data point in proportion to model loss + transform_category = torch.distributions.categorical.Categorical(probabilities) + + popper_net.freeze_weights() + + for condition in range(num_samples): + + index = transform_category.sample() + input_sample = torch.flatten(X_train_tensor[index, :]) + popper_input = Variable(input_sample, requires_grad=True) + + # invert the popper network to determine optimal experiment conditions + for optimization_epoch in range(optimization_epochs): + # feedforward pass on popper network + popper_prediction = popper_net(popper_input) + # compute gradient that maximizes output of popper network + # (i.e. predicted loss of original model) + popper_loss_optim = -popper_prediction + popper_loss_optim.backward() + # compute new input + # with torch.no_grad(): + # delta = -optimization_lr * popper_input.grad + # popper_input += -optimization_lr * popper_input.grad + # print(delta) + # popper_input.grad.zero_() + + with torch.no_grad(): + + # first add repulsion from variable limits + for idx in range(len(input_sample)): + IV_value = input_sample[idx] + IV_limits = IV_limit_list[idx] + dist_to_min = np.abs(IV_value - np.min(IV_limits)) + dist_to_max = np.abs(IV_value - np.max(IV_limits)) + repulsion_from_min = limit_repulsion / (dist_to_min**2) + repulsion_from_max = limit_repulsion / (dist_to_max**2) + IV_value_repulsed = ( + IV_value + repulsion_from_min - repulsion_from_max + ) + popper_input[idx] = IV_value_repulsed + + # now add gradient for theory loss maximization + delta = -optimization_lr * popper_input.grad + popper_input += delta + popper_input.grad.zero_() + + # finally, clip input variable from it's limits + for idx in range(len(input_sample)): + IV_raw_value = input_sample[idx] + IV_limits = IV_limit_list[idx] + IV_clipped_value = np.min( + [IV_raw_value, np.max(IV_limits) - limit_offset] + ) + IV_clipped_value = np.max( + [ + IV_clipped_value, + np.min(IV_limits) + limit_offset, + ] + ) + popper_input[idx] = IV_clipped_value + + # add condition to new experiment sequence + for idx in range(len(input_sample)): + IV_limits = IV_limit_list[idx] + + # first clip value + IV_clipped_value = np.min([IV_raw_value, np.max(IV_limits) - limit_offset]) + IV_clipped_value = np.max( + [IV_clipped_value, np.min(IV_limits) + limit_offset] + ) + # make sure to convert variable to original scale + IV_clipped_sclaled_value = IV_clipped_value + + X[condition, idx] = IV_clipped_sclaled_value + + return X + + +def poppernet_sampler( + X, + model, + X_train, + Y_train, + meta_data, + num_samples: int = 100, + training_epochs: int = 1000, + optimization_epochs=1000, + training_lr: float = 1e-3, + optimization_lr: float = 1e-3, + mse_scale: float = 1, + limit_offset: float = 10**-10, + limit_repulsion: float = 0.000001, + verbose: bool = False, +): + """ + A sampler that returns selected samples for independent variables + that predict the highest loss of the model. The sampler leverages the Popper Net Pooler to + generate a list of ideal samples and then selects samples from the pool X (without replacement) + that are closest to those ideal samples. + + Args: + X: pool of IV conditions to sample from + model: Scikit-learn model, could be either a classification or regression model + X_train: data that the model was trained on + Y_train: labels that the model was trained on + meta_data: Meta-data about the dependent and independent variables + num_samples: number of samples to return + training_epochs: number of epochs to train the popper network for approximating the + error fo the model + optimization_epochs: number of epochs to optimize the samples based on the trained + popper network + training_lr: learning rate for training the popper network + optimization_lr: learning rate for optimizing the samples + mse_scale: scale factor for the MSE loss + limit_offset: a limited offset to prevent the samples from being too close to the value + boundaries + limit_repulsion: a limited repulsion to prevent the samples from being too close to the + allowed value boundaries + verbose: print out the prediction of the popper network as well as its training loss + + Returns: + + """ + + if isinstance(X, Iterable): + X = np.array(list(X)) + + if len(X.shape) == 1: + X = X.reshape(-1, 1) + + if X.shape[0] <= num_samples: + raise Exception("More samples requested than samples available in the pool X.") + + samples = poppernet_pooler( + model, + X_train, + Y_train, + meta_data, + num_samples, + training_epochs, + optimization_epochs, + training_lr, + optimization_lr, + mse_scale, + limit_offset, + limit_repulsion, + verbose, + ) + + X_new = np.empty((num_samples, X.shape[1])) + + # get index of row in X that is closest to each sample + for row, sample in enumerate(samples): + dist = np.linalg.norm(X - sample, axis=1) + idx = np.argmin(dist) + X_new[row, :] = X[idx, :] + X = np.delete(X, idx, axis=0) + + return X_new + + +# define the network +class PopperNet(nn.Module): + def __init__(self, n_input: torch.Tensor, n_output: torch.Tensor): + # Perform initialization of the pytorch superclass + super(PopperNet, self).__init__() + + # Define network layer dimensions + D_in, H1, H2, H3, D_out = [n_input, 64, 64, 64, n_output] + + # Define layer types + self.linear1 = nn.Linear(D_in, H1) + self.linear2 = nn.Linear(H1, H2) + self.linear3 = nn.Linear(H2, H3) + self.linear4 = nn.Linear(H3, D_out) + + def forward(self, x: torch.Tensor): + """ + This method defines the network layering and activation functions + """ + x = self.linear1(x) # hidden layer + x = torch.tanh(x) # activation function + + x = self.linear2(x) # hidden layer + x = torch.tanh(x) # activation function + + x = self.linear3(x) # hidden layer + x = torch.tanh(x) # activation function + + x = self.linear4(x) # output layer + + return x + + def freeze_weights(self): + for param in self.parameters(): + param.requires_grad = False + + +def class_to_onehot(y: np.array, n_classes: int = None): + """Converts a class vector (integers) to binary class matrix. + + E.g. for use with categorical_crossentropy. + + # Arguments + y: class vector to be converted into a matrix + (integers from 0 to num_classes). + n_classes: total number of classes. + + # Returns + A binary matrix representation of the input. + """ + y = np.array(y, dtype="int") + input_shape = y.shape + if input_shape and input_shape[-1] == 1 and len(input_shape) > 1: + input_shape = tuple(input_shape[:-1]) + y = y.ravel() + if not n_classes: + n_classes = np.max(y) + 1 + n = y.shape[0] + categorical = np.zeros((n, n_classes)) + categorical[np.arange(n), y] = 1 + output_shape = input_shape + (n_classes,) + categorical = np.reshape(categorical, output_shape) + return categorical diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py new file mode 100644 index 000000000..f3570b0da --- /dev/null +++ b/tests/test_poppernet_sampler.py @@ -0,0 +1,171 @@ +import numpy as np +import pytest +from sklearn.linear_model import LinearRegression, LogisticRegression + +from autora.experimentalist.sampler.poppernet import poppernet_sampler +from autora.variable import DV, IV, ValueType, VariableCollection + + +def get_xor_data(n: int = 3): + X = ([[1, 0]] * n) + ([[0, 1]] * n) + ([[0, 0]] * n) + ([[1, 1]]) + y = ([0] * n) + ([0] * n) + ([1] * n) + ([1]) + return X, y + + +def get_sin_data(n: int = 100): + x = np.linspace(0, 2 * np.pi, 100) + y = np.sin(x) + return x, y + + +@pytest.fixture +def synthetic_logr_model(): + """ + Creates logistic regression classifier for 3 classes based on synthetic data. + """ + X, y = get_xor_data() + model = LogisticRegression() + model.fit(X, y) + return model + + +@pytest.fixture +def synthetic_linr_model(): + """ + Creates linear regression based on synthetic data. + """ + x, y = get_sin_data() + model = LinearRegression() + model.fit(x.reshape(-1, 1), y) + return model + + +@pytest.fixture +def classification_data_to_test(): + data = np.array( + [ + [1, 0], + [0, 1], + [0, 0], + [1, 1], + ] + ) + return data + + +@pytest.fixture +def regression_data_to_test(): + # data = np.linspace(0, 2 * np.pi, 5) + data = [-10, 0, 1.5, 3, 4.5, 6, 10] + return data + + +def test_poppernet_classification(synthetic_logr_model, classification_data_to_test): + + # Import model and data + X_train, Y_train = get_xor_data() + X = classification_data_to_test + model = synthetic_logr_model + # specify meta data + + # Specify independent variables + iv1 = IV( + name="x", + value_range=(0, 5), + units="intensity", + variable_label="stimulus 1", + ) + + # specify dependent variables + dv1 = DV( + name="y", + value_range=(0, 1), + units="class", + variable_label="class", + type=ValueType.CLASS, + ) + + # Variable collection with ivs and dvs + metadata = VariableCollection( + independent_variables=[iv1, iv1], + dependent_variables=[dv1], + ) + + # Run popper net sampler + samples = poppernet_sampler( + X, + model, + X_train, + Y_train, + metadata, + num_samples=2, + training_epochs=1000, + optimization_epochs=1000, + training_lr=1e-3, + optimization_lr=1e-3, + mse_scale=1, + limit_offset=10**-10, + limit_repulsion=0, + verbose=True, + ) + + print(samples) + # Check that at least one of the resulting samples is the one that is + # underrepresented in the data used for model training + + assert (samples[0, :] == [1, 1]).all or (samples[1, :] == [1, 1]).all + + +def test_poppernet_regression(synthetic_linr_model, regression_data_to_test): + + # Import model and data + X_train, Y_train = get_sin_data() + X = regression_data_to_test + model = synthetic_linr_model + + # specify meta data + + # Specify independent variables + iv = IV( + name="x", + value_range=(0, 2 * np.pi), + units="intensity", + variable_label="stimulus", + ) + + # specify dependent variables + dv = DV( + name="y", + value_range=(-1, 1), + units="real", + variable_label="response", + type=ValueType.REAL, + ) + + # Variable collection with ivs and dvs + metadata = VariableCollection( + independent_variables=[iv], + dependent_variables=[dv], + ) + + # Run popper net sampler + sample = poppernet_sampler( + X, + model, + X_train, + Y_train, + metadata, + num_samples=5, + training_epochs=1000, + optimization_epochs=1000, + training_lr=1e-3, + optimization_lr=1e-3, + mse_scale=1, + limit_offset=10**-10, + limit_repulsion=0, + verbose=True, + ) + + # the first value should be close to one of the local maxima of the + # sine function + assert sample[0] == 1.5 or sample[0] == 4.5 From 894a0887538f64dc9179b4802117596ef2f21b78 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Sat, 3 Dec 2022 19:50:33 -0500 Subject: [PATCH 08/50] minor doc grammar fix --- autora/experimentalist/sampler/poppernet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index dafcfba0f..8dd30fd17 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -24,7 +24,7 @@ def poppernet_pooler( verbose: bool = False, ): """ - A pooler that generates samples for independent variables with the objective to maximize the + A pooler that generates samples for independent variables with the objective of maximizing the (approximated) loss of the model. The samples are generated by first training a neural network to approximate the loss of a model for all patterns in the training data. Once trained, the network is then inverted to generate samples that maximize the approximated loss of the model. From ff50bba8026b31df5213c3d07c9fb06786c8be49 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Sat, 3 Dec 2022 19:58:56 -0500 Subject: [PATCH 09/50] fixed small plotting issue --- autora/experimentalist/sampler/poppernet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 8dd30fd17..84510867e 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -133,7 +133,7 @@ def poppernet_pooler( import matplotlib.pyplot as plt - if len(popper_input.shape) > 0: + if popper_input.shape[1] > 1: plot_input = popper_input[:, 0] plt.scatter(plot_input, popper_target.detach().numpy(), label="target") plt.scatter( From b533ddd809ef33914d02837897b4002380430d2c Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Mon, 5 Dec 2022 10:48:14 -0500 Subject: [PATCH 10/50] added model disagreement sampler and tests --- .../sampler/model_disagreement.py | 72 ++++++++++ tests/test_model_disagreement_sampler.py | 131 ++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 autora/experimentalist/sampler/model_disagreement.py create mode 100644 tests/test_model_disagreement_sampler.py diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py new file mode 100644 index 000000000..1e6a80955 --- /dev/null +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -0,0 +1,72 @@ +from typing import Iterable, List + +import numpy as np + + +def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): + """ + A sampler that returns selected samples for independent variables + for which the models disagree the most in terms of their predictions. + + Args: + X: pool of IV conditions to evaluate in terms of model disagreement + models: List of Scikit-learn (regression or classification) models to compare + num_samples: number of samples to select + + Returns: Sampled pool + """ + + if isinstance(X, Iterable): + X = np.array(list(X)) + + X_predict = np.array(X) + if len(X_predict.shape) == 1: + X_predict = X_predict.reshape(-1, 1) + + model_disagreement = list() + + # collect diagreements for each model apir + for idx_A, model_A in enumerate(models): + for idx_B, model_B in enumerate(models): + + # don't compare the model with itself + if idx_A <= idx_B: + continue + + # determine the prediction method + model_A_predict = getattr(model_A, "predict_proba", None) + if callable(model_A_predict) is False: + model_A_predict = getattr(model_A, "predict", None) + + model_B_predict = getattr(model_B, "predict_proba", None) + if callable(model_B_predict) is False: + model_B_predict = getattr(model_B, "predict", None) + + if model_A_predict is None or model_B_predict is None: + raise Exception("Model must have `predict` or `predict_proba` method.") + + # get predictions from both models + Y_A = model_A_predict(X_predict) + Y_B = model_B_predict(X_predict) + + if Y_A.shape != Y_B.shape: + raise Exception("Models must have same output shape.") + + # determine the disagreement between the two models in terms of mean-squared error + if len(Y_A.shape) == 1: + disagreement = (Y_A - Y_B) ** 2 + else: + disagreement = np.mean((Y_A - Y_B) ** 2, axis=1) + + model_disagreement.append(disagreement) + + if len(model_disagreement) == 0: + raise Exception("No models to compare.") + + # sum up all model disagreements + summed_disagreement = np.sum(model_disagreement, axis=0) + + # sort the summed disagreements + idx = (-summed_disagreement).argsort()[:num_samples] + + return X[idx] diff --git a/tests/test_model_disagreement_sampler.py b/tests/test_model_disagreement_sampler.py new file mode 100644 index 000000000..ed987681e --- /dev/null +++ b/tests/test_model_disagreement_sampler.py @@ -0,0 +1,131 @@ +import numpy as np +import pytest +from sklearn.linear_model import LinearRegression, LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures + +from autora.experimentalist.sampler.model_disagreement import model_disagreement_sampler + + +def get_classification_data(n: int = 100): + x1 = np.linspace(0, 1, n) + x2 = np.linspace(0, 1, n) + + # cross product of x1 and x2 + X = np.array([(x1[i], x2[j]) for i in range(len(x1)) for j in range(len(x2))]) + + # create a vector of 0s and 1s which is 0 whenever x1 < 0.5 and x2 < 0.5 and 1 otherwise + y_A = np.zeros(n * n) + y_B = np.zeros(n * n) + y_A[(X[:, 0] >= 0.5) | (X[:, 1] >= 0.5)] = 1 + y_B[(X[:, 0] >= 0.5)] = 1 + + return X, y_A, y_B + + +def get_polynomial_data(n: int = 100): + x = np.linspace(-1, 1, 100) + y = x**2 + return x, y + + +@pytest.fixture +def synthetic_lr_models(): + """ + Creates two logistic regression classifier for 2 classes based on synthetic data. + Each classifier is trained on a different data set and thus should yield different predictions. + """ + X, y_A, y_B = get_classification_data() + model_A = LogisticRegression() + model_B = LogisticRegression() + model_A.fit(X, y_A) + model_B.fit(X, y_B) + + models = [model_A, model_B] + return models + + +@pytest.fixture +def synthetic_linr_model(): + """ + Creates linear regression based on synthetic data. + """ + x, y = get_polynomial_data() + model = LinearRegression() + model.fit(x.reshape(-1, 1), y) + return model + + +@pytest.fixture +def synthetic_poly_model(): + """ + Creates polynomial regression based on synthetic data. + """ + x, y = get_polynomial_data() + + # define the steps in the pipeline + steps = [ + ( + "poly", + PolynomialFeatures(degree=3), + ), # transform input data into polynomial features + ("lr", LinearRegression()), # fit a linear regression model + ] + # create the pipeline + model = Pipeline(steps) + model.fit(x.reshape(-1, 1), y) + return model + + +@pytest.fixture +def classification_data_to_test(n=10): + x1 = np.linspace(0, 1, n) + x2 = np.linspace(0, 1, n) + + # cross product of x1 and x2 + X = np.array([(x1[i], x2[j]) for i in range(len(x1)) for j in range(len(x2))]) + return X + + +@pytest.fixture +def regression_data_to_test(n=100): + data = np.linspace(-2, 2, n) + return data + + +def test_model_disagreement_classification( + synthetic_lr_models, classification_data_to_test +): + + num_requested_samples = 10 + + # Import model and data + X = classification_data_to_test + models = synthetic_lr_models + + # Run model disagreement sampler + samples = model_disagreement_sampler(X, models, num_requested_samples) + + assert samples.shape[0] == num_requested_samples + assert samples[0, 0] < 0.25 and samples[0, 1] > 0.75 + assert samples[1, 0] < 0.25 and samples[1, 1] > 0.75 + + +def test_model_disagreement_regression( + synthetic_linr_model, synthetic_poly_model, regression_data_to_test +): + + num_requested_samples = 2 + + # Import model and data + X = regression_data_to_test + model_A = synthetic_linr_model + model_B = synthetic_poly_model + models = [model_A, model_B] + + # Run model disagreement sampler + samples = model_disagreement_sampler(X, models, num_requested_samples) + + assert len(samples) == num_requested_samples + assert samples[0] == 2.0 or samples[0] == -2.0 + assert samples[1] == 2.0 or samples[1] == -2.0 From 61b3d83b1924169cca0bef974210309ccd4688ae Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 12:01:55 -0500 Subject: [PATCH 11/50] docs: update dosctring for train_test_filter --- autora/experimentalist/filter.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index 7c106a2a9..c9532589e 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -7,20 +7,34 @@ def weber_filter(values): return filter(lambda s: s[0] <= s[1], values) -def train_test_filter(seed=180, train_p=0.5): +def train_test_filter(seed: int = 180, train_p: float = 0.5): """ A pipeline filter which pseudorandomly assigns values from the input into "train" or "test" - groups. + groups. This is particularly useful when working with streams of data of potentially + unbounded length. + + This isn't a great method for small datasets, as it doesn't guarantee producing training + and test sets which are as close as possible to the specified desired proportions. + + Args: + seed: random number generator seeding value + train_p: proportion of data which go into the training set. A float between 0 and 1. + + Returns: + a tuple of callables `(train_filter, test_filter)` which split the input data + into two complementary streams. + Examples: We can create complementary train and test filters using the function: >>> train_filter, test_filter = train_test_filter(train_p=0.6, seed=180) - The train filter generates a sequence of 60% of the input list. + The `train_filter` generates a sequence of ~60% of the input list – + in this case, 15 of 20 datapoints. >>> list(train_filter(range(20))) [0, 2, 3, 4, 5, 6, 9, 10, 11, 12, 15, 16, 17, 18, 19] - When we run the test_filter, it fills in the gaps. + When we run the `test_filter`, it fills in the gaps, giving us the remaining 5 values: >>> list(test_filter(range(20))) [1, 7, 8, 13, 14] @@ -33,6 +47,9 @@ def train_test_filter(seed=180, train_p=0.5): >>> list(train_filter(range(40, 50))) [41, 42, 44, 45, 46, 49] + As the number of samples grows, the fraction in the train and test sets + will approach `train_p` and `1 - train_p`. + The test_filter fills in the gaps again. >>> list(test_filter(range(20, 30))) [21, 24, 25, 26] @@ -47,7 +64,6 @@ def train_test_filter(seed=180, train_p=0.5): >>> list(test_filter_regenerated(range(20))) [1, 7, 8, 13, 14] - It also works on tuple-valued lists: >>> from itertools import product >>> train_filter_tuple, test_filter_tuple = train_test_filter(train_p=0.3, seed=42) From 8b9c70df5bb8b7f68ac2f2478f09e68f05fa8836 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 12:08:42 -0500 Subject: [PATCH 12/50] docs: add types to train_test_filter output --- autora/experimentalist/filter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index c9532589e..ff183fa4e 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import Callable, Iterable, Tuple import numpy as np @@ -7,7 +8,9 @@ def weber_filter(values): return filter(lambda s: s[0] <= s[1], values) -def train_test_filter(seed: int = 180, train_p: float = 0.5): +def train_test_filter( + seed: int = 180, train_p: float = 0.5 +) -> Tuple[Callable[[Iterable], Iterable], Callable[[Iterable], Iterable]]: """ A pipeline filter which pseudorandomly assigns values from the input into "train" or "test" groups. This is particularly useful when working with streams of data of potentially @@ -22,7 +25,7 @@ def train_test_filter(seed: int = 180, train_p: float = 0.5): Returns: a tuple of callables `(train_filter, test_filter)` which split the input data - into two complementary streams. + into two complementary streams. Examples: From a4b6f0cb498d79ebce5ce51e6f9768e334b20d71 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 12:15:11 -0500 Subject: [PATCH 13/50] docs: update docstring within the train_test_filter --- autora/experimentalist/filter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index ff183fa4e..8d56af435 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -99,15 +99,21 @@ def train_test_filter( _TrainTest = Enum("_TrainTest", ["train", "test"]) - def _train_test_stream(): + def train_test_stream(): + """Generates a pseudorandom stream of _TrainTest.train and _TrainTest.test.""" rng = np.random.default_rng(seed) while True: yield rng.choice([_TrainTest.train, _TrainTest.test], p=(train_p, test_p)) def _factory(allow): - _stream = _train_test_stream() + """Factory to make complementary generators which split their input + corresponding to the values of the pseudorandom train_test_stream.""" + _stream = train_test_stream() def _generator(values): + """Generator which yields items from the `values` depending on + whether the corresponding item from the `_stream` + matches the `allow` parameter.""" for v, train_test in zip(values, _stream): if train_test == allow: yield v From 1c032cdc5355e5a3ef4df97f31cc538181382449 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:28:36 -0500 Subject: [PATCH 14/50] docs: add reference to the scikit-learn train_test_split --- autora/experimentalist/filter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index 8d56af435..f67bc150e 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -18,6 +18,8 @@ def train_test_filter( This isn't a great method for small datasets, as it doesn't guarantee producing training and test sets which are as close as possible to the specified desired proportions. + Consider using the scikit-learn `train_test_split` for cases where it's practical to + enumerate the full dataset in advance. Args: seed: random number generator seeding value From b6d36bdbfcbc72e4650b67320560f32489a9ad54 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:40:44 -0500 Subject: [PATCH 15/50] refactor: move popper diagnostics plot to separate function --- autora/experimentalist/sampler/poppernet.py | 50 +++++++++------------ 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 8dd30fd17..d56d26ccf 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -21,7 +21,6 @@ def poppernet_pooler( mse_scale: float = 1, limit_offset: float = 10**-10, limit_repulsion: float = 0, - verbose: bool = False, ): """ A pooler that generates samples for independent variables with the objective of maximizing the @@ -128,32 +127,6 @@ def poppernet_pooler( popper_optimizer.step() losses.append(loss.item()) - if verbose: - print("Finished training Popper Network...") - - import matplotlib.pyplot as plt - - if len(popper_input.shape) > 0: - plot_input = popper_input[:, 0] - plt.scatter(plot_input, popper_target.detach().numpy(), label="target") - plt.scatter( - plot_input, popper_prediction.detach().numpy(), label="prediction" - ) - else: - plot_input = popper_input - plt.plot(plot_input, popper_target.detach().numpy(), label="target") - plt.plot(plot_input, popper_prediction.detach().numpy(), label="prediction") - - plt.xlabel("x") - plt.ylabel("y") - plt.legend() - plt.show() - - plt.plot(losses) - plt.xlabel("epoch") - plt.ylabel("loss") - plt.show() - # now that the popper network is trained we can sample new data points # to sample data points we need to provide the popper network with an initial condition # we will sample those initial conditions proportional to the loss of the current model @@ -240,6 +213,28 @@ def poppernet_pooler( return X +def plot_popper_diagnostics(losses, popper_input, popper_prediction, popper_target): + print("Finished training Popper Network...") + import matplotlib.pyplot as plt + + if len(popper_input.shape) > 0: + plot_input = popper_input[:, 0] + plt.scatter(plot_input, popper_target.detach().numpy(), label="target") + plt.scatter(plot_input, popper_prediction.detach().numpy(), label="prediction") + else: + plot_input = popper_input + plt.plot(plot_input, popper_target.detach().numpy(), label="target") + plt.plot(plot_input, popper_prediction.detach().numpy(), label="prediction") + plt.xlabel("x") + plt.ylabel("y") + plt.legend() + plt.show() + plt.plot(losses) + plt.xlabel("epoch") + plt.ylabel("loss") + plt.show() + + def poppernet_sampler( X, model, @@ -308,7 +303,6 @@ def poppernet_sampler( mse_scale, limit_offset, limit_repulsion, - verbose, ) X_new = np.empty((num_samples, X.shape[1])) From de81e245639a321c46137f2f4f51c35914f9893c Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:41:11 -0500 Subject: [PATCH 16/50] refactor: make function argument names explicit --- autora/experimentalist/sampler/poppernet.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index d56d26ccf..7c45a6a48 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -291,18 +291,18 @@ def poppernet_sampler( raise Exception("More samples requested than samples available in the pool X.") samples = poppernet_pooler( - model, - X_train, - Y_train, - meta_data, - num_samples, - training_epochs, - optimization_epochs, - training_lr, - optimization_lr, - mse_scale, - limit_offset, - limit_repulsion, + model=model, + X_train=X_train, + Y_train=Y_train, + meta_data=meta_data, + num_samples=num_samples, + training_epochs=training_epochs, + optimization_epochs=optimization_epochs, + training_lr=training_lr, + optimization_lr=optimization_lr, + mse_scale=mse_scale, + limit_offset=limit_offset, + limit_repulsion=limit_repulsion, ) X_new = np.empty((num_samples, X.shape[1])) From 89fa9f7c7a48b822148c562d56c61012621fc55e Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:44:15 -0500 Subject: [PATCH 17/50] =?UTF-8?q?refactor:=20rename=20x=20and=20y=20to=20l?= =?UTF-8?q?owercase=20=E2=80=93=20python=20style?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- autora/experimentalist/sampler/poppernet.py | 76 ++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 7c45a6a48..fe943177b 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -10,8 +10,8 @@ def poppernet_pooler( model, - X_train: np.ndarray, - Y_train: np.ndarray, + x_train: np.ndarray, + y_train: np.ndarray, meta_data: VariableCollection, num_samples: int = 100, training_epochs: int = 1000, @@ -33,8 +33,8 @@ def poppernet_pooler( Args: model: Scikit-learn model, could be either a classification or regression model - X_train: data that the model was trained on - Y_train: labels that the model was trained on + x_train: data that the model was trained on + y_train: labels that the model was trained on meta_data: Meta-data about the dependent and independent variables num_samples: number of samples to return training_epochs: number of epochs to train the popper network for approximating the @@ -56,22 +56,22 @@ def poppernet_pooler( # format input - X_train = np.array(X_train) - if len(X_train.shape) == 1: - X_train = X_train.reshape(-1, 1) + x_train = np.array(x_train) + if len(x_train.shape) == 1: + x_train = x_train.reshape(-1, 1) - X = np.empty([num_samples, X_train.shape[1]]) + X = np.empty([num_samples, x_train.shape[1]]) - Y_train = np.array(Y_train) - if len(Y_train.shape) == 1: - Y_train = Y_train.reshape(-1, 1) + y_train = np.array(y_train) + if len(y_train.shape) == 1: + y_train = y_train.reshape(-1, 1) if meta_data.dependent_variables[0].type == ValueType.CLASS: - # find all unique values in Y_train - num_classes = len(np.unique(Y_train)) - Y_train = class_to_onehot(Y_train, n_classes=num_classes) + # find all unique values in y_train + num_classes = len(np.unique(y_train)) + y_train = class_to_onehot(y_train, n_classes=num_classes) - X_train_tensor = torch.from_numpy(X_train).float() + X_train_tensor = torch.from_numpy(x_train).float() # create list of IV limits IVs = meta_data.independent_variables @@ -88,7 +88,7 @@ def poppernet_pooler( n_output = len(meta_data.dependent_variables) # get input pattern for popper net - popper_input = Variable(torch.from_numpy(X_train), requires_grad=False).float() + popper_input = Variable(torch.from_numpy(x_train), requires_grad=False).float() # get target pattern for popper net model_predict = getattr(model, "predict_proba", None) @@ -98,10 +98,10 @@ def poppernet_pooler( if callable(model_predict) is False or model_predict is None: raise Exception("Model must have `predict` or `predict_proba` method.") - model_prediction = model_predict(X_train) + model_prediction = model_predict(x_train) criterion = nn.MSELoss() - model_loss = (model_prediction - Y_train) ** 2 * mse_scale + model_loss = (model_prediction - y_train) ** 2 * mse_scale model_loss = np.mean(model_loss, axis=1) model_loss = torch.from_numpy(model_loss).float() popper_target = Variable(model_loss, requires_grad=False) @@ -236,10 +236,10 @@ def plot_popper_diagnostics(losses, popper_input, popper_prediction, popper_targ def poppernet_sampler( - X, + x, model, - X_train, - Y_train, + x_train, + y_train, meta_data, num_samples: int = 100, training_epochs: int = 1000, @@ -258,10 +258,10 @@ def poppernet_sampler( that are closest to those ideal samples. Args: - X: pool of IV conditions to sample from + x: pool of IV conditions to sample from model: Scikit-learn model, could be either a classification or regression model - X_train: data that the model was trained on - Y_train: labels that the model was trained on + x_train: data that the model was trained on + y_train: labels that the model was trained on meta_data: Meta-data about the dependent and independent variables num_samples: number of samples to return training_epochs: number of epochs to train the popper network for approximating the @@ -281,19 +281,19 @@ def poppernet_sampler( """ - if isinstance(X, Iterable): - X = np.array(list(X)) + if isinstance(x, Iterable): + x = np.array(list(x)) - if len(X.shape) == 1: - X = X.reshape(-1, 1) + if len(x.shape) == 1: + x = x.reshape(-1, 1) - if X.shape[0] <= num_samples: - raise Exception("More samples requested than samples available in the pool X.") + if x.shape[0] <= num_samples: + raise Exception("More samples requested than samples available in the pool x.") samples = poppernet_pooler( model=model, - X_train=X_train, - Y_train=Y_train, + x_train=x_train, + y_train=y_train, meta_data=meta_data, num_samples=num_samples, training_epochs=training_epochs, @@ -305,16 +305,16 @@ def poppernet_sampler( limit_repulsion=limit_repulsion, ) - X_new = np.empty((num_samples, X.shape[1])) + x_new = np.empty((num_samples, x.shape[1])) - # get index of row in X that is closest to each sample + # get index of row in x that is closest to each sample for row, sample in enumerate(samples): - dist = np.linalg.norm(X - sample, axis=1) + dist = np.linalg.norm(x - sample, axis=1) idx = np.argmin(dist) - X_new[row, :] = X[idx, :] - X = np.delete(X, idx, axis=0) + x_new[row, :] = x[idx, :] + x = np.delete(x, idx, axis=0) - return X_new + return x_new # define the network From 70d2660568855202e5a1f518f61e50b978d53e27 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:45:32 -0500 Subject: [PATCH 18/50] =?UTF-8?q?refactor:=20rename=20x=20and=20iv=20to=20?= =?UTF-8?q?lowercase=20=E2=80=93=20python=20style?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- autora/experimentalist/sampler/poppernet.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index fe943177b..38dc2ad45 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -71,17 +71,17 @@ def poppernet_pooler( num_classes = len(np.unique(y_train)) y_train = class_to_onehot(y_train, n_classes=num_classes) - X_train_tensor = torch.from_numpy(x_train).float() + x_train_tensor = torch.from_numpy(x_train).float() # create list of IV limits - IVs = meta_data.independent_variables - IV_limit_list = list() - for IV in IVs: - if hasattr(IV, "value_range"): - value_range = cast(Tuple, IV.value_range) + ivs = meta_data.independent_variables + iv_limit_list = list() + for iv in ivs: + if hasattr(iv, "value_range"): + value_range = cast(Tuple, iv.value_range) lower_bound = value_range[0] upper_bound = value_range[1] - IV_limit_list.append(([lower_bound, upper_bound])) + iv_limit_list.append(([lower_bound, upper_bound])) # get dimensions of input and output n_input = len(meta_data.independent_variables) @@ -143,7 +143,7 @@ def poppernet_pooler( for condition in range(num_samples): index = transform_category.sample() - input_sample = torch.flatten(X_train_tensor[index, :]) + input_sample = torch.flatten(x_train_tensor[index, :]) popper_input = Variable(input_sample, requires_grad=True) # invert the popper network to determine optimal experiment conditions @@ -166,7 +166,7 @@ def poppernet_pooler( # first add repulsion from variable limits for idx in range(len(input_sample)): IV_value = input_sample[idx] - IV_limits = IV_limit_list[idx] + IV_limits = iv_limit_list[idx] dist_to_min = np.abs(IV_value - np.min(IV_limits)) dist_to_max = np.abs(IV_value - np.max(IV_limits)) repulsion_from_min = limit_repulsion / (dist_to_min**2) @@ -184,7 +184,7 @@ def poppernet_pooler( # finally, clip input variable from it's limits for idx in range(len(input_sample)): IV_raw_value = input_sample[idx] - IV_limits = IV_limit_list[idx] + IV_limits = iv_limit_list[idx] IV_clipped_value = np.min( [IV_raw_value, np.max(IV_limits) - limit_offset] ) @@ -198,7 +198,7 @@ def poppernet_pooler( # add condition to new experiment sequence for idx in range(len(input_sample)): - IV_limits = IV_limit_list[idx] + IV_limits = iv_limit_list[idx] # first clip value IV_clipped_value = np.min([IV_raw_value, np.max(IV_limits) - limit_offset]) From 5e0b37420bbb8928cbf5c4fde8e5e5ca962f89d3 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:45:52 -0500 Subject: [PATCH 19/50] =?UTF-8?q?refactor:=20rename=20iv=20to=20lowercase?= =?UTF-8?q?=20=E2=80=93=20python=20style?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- autora/experimentalist/sampler/poppernet.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 38dc2ad45..c3c0dd81a 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -165,14 +165,14 @@ def poppernet_pooler( # first add repulsion from variable limits for idx in range(len(input_sample)): - IV_value = input_sample[idx] - IV_limits = iv_limit_list[idx] - dist_to_min = np.abs(IV_value - np.min(IV_limits)) - dist_to_max = np.abs(IV_value - np.max(IV_limits)) + iv_value = input_sample[idx] + iv_limits = iv_limit_list[idx] + dist_to_min = np.abs(iv_value - np.min(iv_limits)) + dist_to_max = np.abs(iv_value - np.max(iv_limits)) repulsion_from_min = limit_repulsion / (dist_to_min**2) repulsion_from_max = limit_repulsion / (dist_to_max**2) IV_value_repulsed = ( - IV_value + repulsion_from_min - repulsion_from_max + iv_value + repulsion_from_min - repulsion_from_max ) popper_input[idx] = IV_value_repulsed @@ -184,26 +184,26 @@ def poppernet_pooler( # finally, clip input variable from it's limits for idx in range(len(input_sample)): IV_raw_value = input_sample[idx] - IV_limits = iv_limit_list[idx] + iv_limits = iv_limit_list[idx] IV_clipped_value = np.min( - [IV_raw_value, np.max(IV_limits) - limit_offset] + [IV_raw_value, np.max(iv_limits) - limit_offset] ) IV_clipped_value = np.max( [ IV_clipped_value, - np.min(IV_limits) + limit_offset, + np.min(iv_limits) + limit_offset, ] ) popper_input[idx] = IV_clipped_value # add condition to new experiment sequence for idx in range(len(input_sample)): - IV_limits = iv_limit_list[idx] + iv_limits = iv_limit_list[idx] # first clip value - IV_clipped_value = np.min([IV_raw_value, np.max(IV_limits) - limit_offset]) + IV_clipped_value = np.min([IV_raw_value, np.max(iv_limits) - limit_offset]) IV_clipped_value = np.max( - [IV_clipped_value, np.min(IV_limits) + limit_offset] + [IV_clipped_value, np.min(iv_limits) + limit_offset] ) # make sure to convert variable to original scale IV_clipped_sclaled_value = IV_clipped_value From 6732477d22321413754f7748c6d665c1081a15ea Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:46:06 -0500 Subject: [PATCH 20/50] =?UTF-8?q?refactor:=20rename=20iv=20to=20lowercase?= =?UTF-8?q?=20=E2=80=93=20python=20style?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- autora/experimentalist/sampler/poppernet.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index c3c0dd81a..42f377fb5 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -183,30 +183,30 @@ def poppernet_pooler( # finally, clip input variable from it's limits for idx in range(len(input_sample)): - IV_raw_value = input_sample[idx] + iv_raw_value = input_sample[idx] iv_limits = iv_limit_list[idx] - IV_clipped_value = np.min( - [IV_raw_value, np.max(iv_limits) - limit_offset] + iv_clipped_value = np.min( + [iv_raw_value, np.max(iv_limits) - limit_offset] ) - IV_clipped_value = np.max( + iv_clipped_value = np.max( [ - IV_clipped_value, + iv_clipped_value, np.min(iv_limits) + limit_offset, ] ) - popper_input[idx] = IV_clipped_value + popper_input[idx] = iv_clipped_value # add condition to new experiment sequence for idx in range(len(input_sample)): iv_limits = iv_limit_list[idx] # first clip value - IV_clipped_value = np.min([IV_raw_value, np.max(iv_limits) - limit_offset]) - IV_clipped_value = np.max( - [IV_clipped_value, np.min(iv_limits) + limit_offset] + iv_clipped_value = np.min([iv_raw_value, np.max(iv_limits) - limit_offset]) + iv_clipped_value = np.max( + [iv_clipped_value, np.min(iv_limits) + limit_offset] ) # make sure to convert variable to original scale - IV_clipped_sclaled_value = IV_clipped_value + IV_clipped_sclaled_value = iv_clipped_value X[condition, idx] = IV_clipped_sclaled_value From 1b833d5d77940ef52c7c3008d281ff59e5cdc862 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:46:16 -0500 Subject: [PATCH 21/50] =?UTF-8?q?refactor:=20rename=20iv=20to=20lowercase?= =?UTF-8?q?=20=E2=80=93=20python=20style?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- autora/experimentalist/sampler/poppernet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 42f377fb5..36e39ef35 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -206,9 +206,9 @@ def poppernet_pooler( [iv_clipped_value, np.min(iv_limits) + limit_offset] ) # make sure to convert variable to original scale - IV_clipped_sclaled_value = iv_clipped_value + iv_clipped_sclaled_value = iv_clipped_value - X[condition, idx] = IV_clipped_sclaled_value + X[condition, idx] = iv_clipped_sclaled_value return X From 8c4747345a6b680630b368d715a0b5e0435fe3b0 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:46:59 -0500 Subject: [PATCH 22/50] refactor: fix typo --- autora/experimentalist/sampler/poppernet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 36e39ef35..d1941a2c5 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -206,9 +206,9 @@ def poppernet_pooler( [iv_clipped_value, np.min(iv_limits) + limit_offset] ) # make sure to convert variable to original scale - iv_clipped_sclaled_value = iv_clipped_value + iv_clipped_scaled_value = iv_clipped_value - X[condition, idx] = iv_clipped_sclaled_value + X[condition, idx] = iv_clipped_scaled_value return X From 1f03301216d9d7a4c126e9afd8558595a5e9e3de Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:47:32 -0500 Subject: [PATCH 23/50] refactor: x lowercase --- autora/experimentalist/sampler/poppernet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index d1941a2c5..7af2fcbaa 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -60,7 +60,7 @@ def poppernet_pooler( if len(x_train.shape) == 1: x_train = x_train.reshape(-1, 1) - X = np.empty([num_samples, x_train.shape[1]]) + x = np.empty([num_samples, x_train.shape[1]]) y_train = np.array(y_train) if len(y_train.shape) == 1: @@ -208,9 +208,9 @@ def poppernet_pooler( # make sure to convert variable to original scale iv_clipped_scaled_value = iv_clipped_value - X[condition, idx] = iv_clipped_scaled_value + x[condition, idx] = iv_clipped_scaled_value - return X + return x def plot_popper_diagnostics(losses, popper_input, popper_prediction, popper_target): From 9c8a28e269dfabf28918edadac1a99778a8a9f2b Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:47:52 -0500 Subject: [PATCH 24/50] refactor: fix typo --- autora/experimentalist/sampler/poppernet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 7af2fcbaa..2a1f927e4 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -181,7 +181,7 @@ def poppernet_pooler( popper_input += delta popper_input.grad.zero_() - # finally, clip input variable from it's limits + # finally, clip input variable from its limits for idx in range(len(input_sample)): iv_raw_value = input_sample[idx] iv_limits = iv_limit_list[idx] From e729759f9d4a5fc7eb749a3f0487192cea484433 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:48:07 -0500 Subject: [PATCH 25/50] refactor: iv to lowercase --- autora/experimentalist/sampler/poppernet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 2a1f927e4..e8ed415b3 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -171,10 +171,10 @@ def poppernet_pooler( dist_to_max = np.abs(iv_value - np.max(iv_limits)) repulsion_from_min = limit_repulsion / (dist_to_min**2) repulsion_from_max = limit_repulsion / (dist_to_max**2) - IV_value_repulsed = ( + iv_value_repulsed = ( iv_value + repulsion_from_min - repulsion_from_max ) - popper_input[idx] = IV_value_repulsed + popper_input[idx] = iv_value_repulsed # now add gradient for theory loss maximization delta = -optimization_lr * popper_input.grad From b1ebfe42126a85f2152969a67e9d3d9cd3a99eca Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:48:30 -0500 Subject: [PATCH 26/50] refactor: typo --- autora/experimentalist/sampler/poppernet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index e8ed415b3..cc81eafce 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -131,7 +131,7 @@ def poppernet_pooler( # to sample data points we need to provide the popper network with an initial condition # we will sample those initial conditions proportional to the loss of the current model - # feed avarage model losses through softmax + # feed average model losses through softmax # model_loss_avg= torch.from_numpy(np.mean(model_loss.detach().numpy(), axis=1)).float() softmax_func = torch.nn.Softmax(dim=0) probabilities = softmax_func(model_loss) From fd5d7f74367bfac631c275c7a144602cb45e8a5a Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 14:51:26 -0500 Subject: [PATCH 27/50] docs: remove lying comment --- autora/experimentalist/sampler/poppernet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index cc81eafce..2f1ada2d8 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -119,7 +119,7 @@ def poppernet_pooler( # train the network losses = [] - for epoch in range(training_epochs): # train for 10 epochs + for epoch in range(training_epochs): popper_prediction = popper_net(popper_input) loss = criterion(popper_prediction, popper_target.reshape(-1, 1)) popper_optimizer.zero_grad() From cc7fe41068eb50457a74be32ec3491f0302ef8aa Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 15:11:33 -0500 Subject: [PATCH 28/50] refactor: rename metadata for consistency --- autora/experimentalist/sampler/poppernet.py | 84 ++++++--------------- 1 file changed, 22 insertions(+), 62 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 2f1ada2d8..9e52ac27c 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -12,7 +12,7 @@ def poppernet_pooler( model, x_train: np.ndarray, y_train: np.ndarray, - meta_data: VariableCollection, + metadata: VariableCollection, num_samples: int = 100, training_epochs: int = 1000, optimization_epochs: int = 1000, @@ -35,7 +35,7 @@ def poppernet_pooler( model: Scikit-learn model, could be either a classification or regression model x_train: data that the model was trained on y_train: labels that the model was trained on - meta_data: Meta-data about the dependent and independent variables + metadata: Meta-data about the dependent and independent variables num_samples: number of samples to return training_epochs: number of epochs to train the popper network for approximating the error fo the model @@ -66,7 +66,7 @@ def poppernet_pooler( if len(y_train.shape) == 1: y_train = y_train.reshape(-1, 1) - if meta_data.dependent_variables[0].type == ValueType.CLASS: + if metadata.dependent_variables[0].type == ValueType.CLASS: # find all unique values in y_train num_classes = len(np.unique(y_train)) y_train = class_to_onehot(y_train, n_classes=num_classes) @@ -74,7 +74,7 @@ def poppernet_pooler( x_train_tensor = torch.from_numpy(x_train).float() # create list of IV limits - ivs = meta_data.independent_variables + ivs = metadata.independent_variables iv_limit_list = list() for iv in ivs: if hasattr(iv, "value_range"): @@ -84,8 +84,8 @@ def poppernet_pooler( iv_limit_list.append(([lower_bound, upper_bound])) # get dimensions of input and output - n_input = len(meta_data.independent_variables) - n_output = len(meta_data.dependent_variables) + n_input = len(metadata.independent_variables) + n_output = len(metadata.dependent_variables) # get input pattern for popper net popper_input = Variable(torch.from_numpy(x_train), requires_grad=False).float() @@ -236,20 +236,8 @@ def plot_popper_diagnostics(losses, popper_input, popper_prediction, popper_targ def poppernet_sampler( - x, - model, - x_train, - y_train, - meta_data, - num_samples: int = 100, - training_epochs: int = 1000, - optimization_epochs=1000, - training_lr: float = 1e-3, - optimization_lr: float = 1e-3, - mse_scale: float = 1, - limit_offset: float = 10**-10, - limit_repulsion: float = 0.000001, - verbose: bool = False, + samples, + allowed_values, ): """ A sampler that returns selected samples for independent variables @@ -258,61 +246,33 @@ def poppernet_sampler( that are closest to those ideal samples. Args: - x: pool of IV conditions to sample from - model: Scikit-learn model, could be either a classification or regression model - x_train: data that the model was trained on - y_train: labels that the model was trained on - meta_data: Meta-data about the dependent and independent variables - num_samples: number of samples to return - training_epochs: number of epochs to train the popper network for approximating the - error fo the model - optimization_epochs: number of epochs to optimize the samples based on the trained - popper network - training_lr: learning rate for training the popper network - optimization_lr: learning rate for optimizing the samples - mse_scale: scale factor for the MSE loss - limit_offset: a limited offset to prevent the samples from being too close to the value - boundaries - limit_repulsion: a limited repulsion to prevent the samples from being too close to the - allowed value boundaries - verbose: print out the prediction of the popper network as well as its training loss + samples: output from the poppernet_pooler + allowed_samples: allowed values of IVs conditions to sample from Returns: + the nearest values from `allowed_samples` to the `samples` """ - if isinstance(x, Iterable): - x = np.array(list(x)) + if isinstance(allowed_values, Iterable): + allowed_values = np.array(list(allowed_values)) + + if len(allowed_values.shape) == 1: + allowed_values = allowed_values.reshape(-1, 1) - if len(x.shape) == 1: - x = x.reshape(-1, 1) + num_samples = samples.shape[0] - if x.shape[0] <= num_samples: + if allowed_values.shape[0] <= num_samples: raise Exception("More samples requested than samples available in the pool x.") - samples = poppernet_pooler( - model=model, - x_train=x_train, - y_train=y_train, - meta_data=meta_data, - num_samples=num_samples, - training_epochs=training_epochs, - optimization_epochs=optimization_epochs, - training_lr=training_lr, - optimization_lr=optimization_lr, - mse_scale=mse_scale, - limit_offset=limit_offset, - limit_repulsion=limit_repulsion, - ) - - x_new = np.empty((num_samples, x.shape[1])) + x_new = np.empty((num_samples, allowed_values.shape[1])) # get index of row in x that is closest to each sample for row, sample in enumerate(samples): - dist = np.linalg.norm(x - sample, axis=1) + dist = np.linalg.norm(allowed_values - sample, axis=1) idx = np.argmin(dist) - x_new[row, :] = x[idx, :] - x = np.delete(x, idx, axis=0) + x_new[row, :] = allowed_values[idx, :] + allowed_values = np.delete(allowed_values, idx, axis=0) return x_new From a09a17e12d61350418e355ef1cc6e75c54602b53 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 15:12:09 -0500 Subject: [PATCH 29/50] refactor: add test for pooler and sampler using pipeline --- tests/test_poppernet_sampler.py | 40 +++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py index f3570b0da..d30fdb5fa 100644 --- a/tests/test_poppernet_sampler.py +++ b/tests/test_poppernet_sampler.py @@ -1,8 +1,9 @@ import numpy as np import pytest +from experimentalist.pipeline import Pipeline from sklearn.linear_model import LinearRegression, LogisticRegression -from autora.experimentalist.sampler.poppernet import poppernet_sampler +from autora.experimentalist.sampler.poppernet import poppernet_pooler, poppernet_sampler from autora.variable import DV, IV, ValueType, VariableCollection @@ -92,23 +93,29 @@ def test_poppernet_classification(synthetic_logr_model, classification_data_to_t ) # Run popper net sampler - samples = poppernet_sampler( - X, - model, - X_train, - Y_train, - metadata, - num_samples=2, - training_epochs=1000, - optimization_epochs=1000, - training_lr=1e-3, - optimization_lr=1e-3, - mse_scale=1, - limit_offset=10**-10, - limit_repulsion=0, - verbose=True, + poppernet_pipeline = Pipeline( + [("pool", poppernet_pooler), ("sampler", poppernet_sampler)], + params={ + "pool": dict( + model=model, + x_train=X_train, + y_train=Y_train, + metadata=metadata, + num_samples=2, + training_epochs=1000, + optimization_epochs=1000, + training_lr=1e-3, + optimization_lr=1e-3, + mse_scale=1, + limit_offset=10**-10, + limit_repulsion=0, + ), + "sampler": {"allowed_values": X}, + }, ) + samples = poppernet_pipeline.run() + print(samples) # Check that at least one of the resulting samples is the one that is # underrepresented in the data used for model training @@ -163,7 +170,6 @@ def test_poppernet_regression(synthetic_linr_model, regression_data_to_test): mse_scale=1, limit_offset=10**-10, limit_repulsion=0, - verbose=True, ) # the first value should be close to one of the local maxima of the From 24264433c353db1c74f5f9c275fa80cb3a3c3d62 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 15:15:28 -0500 Subject: [PATCH 30/50] refactor: use pipeline for regression sampler --- tests/test_poppernet_sampler.py | 36 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py index d30fdb5fa..edab22309 100644 --- a/tests/test_poppernet_sampler.py +++ b/tests/test_poppernet_sampler.py @@ -155,23 +155,29 @@ def test_poppernet_regression(synthetic_linr_model, regression_data_to_test): dependent_variables=[dv], ) - # Run popper net sampler - sample = poppernet_sampler( - X, - model, - X_train, - Y_train, - metadata, - num_samples=5, - training_epochs=1000, - optimization_epochs=1000, - training_lr=1e-3, - optimization_lr=1e-3, - mse_scale=1, - limit_offset=10**-10, - limit_repulsion=0, + poppernet_pipeline = Pipeline( + [("pool", poppernet_pooler), ("sampler", poppernet_sampler)], + params={ + "pool": dict( + model=model, + x_train=X_train, + y_train=Y_train, + metadata=metadata, + num_samples=5, + training_epochs=1000, + optimization_epochs=1000, + training_lr=1e-3, + optimization_lr=1e-3, + mse_scale=1, + limit_offset=10**-10, + limit_repulsion=0, + ), + "sampler": {"allowed_values": X}, + }, ) + sample = poppernet_pipeline.run() + # the first value should be close to one of the local maxima of the # sine function assert sample[0] == 1.5 or sample[0] == 4.5 From cb8a8a3a6a62ca92bb381b2b01adfccb886e1af2 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 5 Dec 2022 15:27:46 -0500 Subject: [PATCH 31/50] refactor: rename nearest_values_sampler --- autora/experimentalist/sampler/poppernet.py | 12 +++++------- tests/test_poppernet_sampler.py | 9 ++++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index 9e52ac27c..21f40c87c 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -235,19 +235,17 @@ def plot_popper_diagnostics(losses, popper_input, popper_prediction, popper_targ plt.show() -def poppernet_sampler( +def nearest_values_sampler( samples, allowed_values, ): """ - A sampler that returns selected samples for independent variables - that predict the highest loss of the model. The sampler leverages the Popper Net Pooler to - generate a list of ideal samples and then selects samples from the pool X (without replacement) - that are closest to those ideal samples. + A sampler which returns the nearest values between the input samples and the allowed values, + without replacement. Args: - samples: output from the poppernet_pooler - allowed_samples: allowed values of IVs conditions to sample from + samples: input conditions + allowed_samples: allowed conditions to sample from Returns: the nearest values from `allowed_samples` to the `samples` diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py index edab22309..532e3fc83 100644 --- a/tests/test_poppernet_sampler.py +++ b/tests/test_poppernet_sampler.py @@ -3,7 +3,10 @@ from experimentalist.pipeline import Pipeline from sklearn.linear_model import LinearRegression, LogisticRegression -from autora.experimentalist.sampler.poppernet import poppernet_pooler, poppernet_sampler +from autora.experimentalist.sampler.poppernet import ( + nearest_values_sampler, + poppernet_pooler, +) from autora.variable import DV, IV, ValueType, VariableCollection @@ -94,7 +97,7 @@ def test_poppernet_classification(synthetic_logr_model, classification_data_to_t # Run popper net sampler poppernet_pipeline = Pipeline( - [("pool", poppernet_pooler), ("sampler", poppernet_sampler)], + [("pool", poppernet_pooler), ("sampler", nearest_values_sampler)], params={ "pool": dict( model=model, @@ -156,7 +159,7 @@ def test_poppernet_regression(synthetic_linr_model, regression_data_to_test): ) poppernet_pipeline = Pipeline( - [("pool", poppernet_pooler), ("sampler", poppernet_sampler)], + [("pool", poppernet_pooler), ("sampler", nearest_values_sampler)], params={ "pool": dict( model=model, From bc0d4f57b6bf75dd0874d15a10edccd57234d7a7 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 13:31:53 -0500 Subject: [PATCH 32/50] fix: convert OPS dictionary into a factory function --- autora/theorist/darts/model_search.py | 4 +- autora/theorist/darts/operations.py | 190 +++++++++++++++----------- 2 files changed, 112 insertions(+), 82 deletions(-) diff --git a/autora/theorist/darts/model_search.py b/autora/theorist/darts/model_search.py index 490675912..10e4c2640 100755 --- a/autora/theorist/darts/model_search.py +++ b/autora/theorist/darts/model_search.py @@ -11,11 +11,11 @@ from autora.theorist.darts.fan_out import Fan_Out from autora.theorist.darts.operations import ( - OPS, PRIMITIVES, Genotype, get_operation_label, isiterable, + operation_factory, ) @@ -58,7 +58,7 @@ def __init__(self, primitives: Sequence[str] = PRIMITIVES): # loop through all the 8 primitive operations for primitive in primitives: # OPS returns an nn module for a given primitive (defines as a string) - op = OPS[primitive] + op = operation_factory(primitive) # add the operation self._ops.append(op) diff --git a/autora/theorist/darts/operations.py b/autora/theorist/darts/operations.py index 69c9e5fd0..05603b04b 100755 --- a/autora/theorist/darts/operations.py +++ b/autora/theorist/darts/operations.py @@ -538,85 +538,115 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # defines all the operations. affine is turned off for cuda (optimization prposes) -OPS = { - "none": Zero(1), - "add": nn.Sequential(Identity()), - "subtract": nn.Sequential(NegIdentity()), - "mult": nn.Sequential( - nn.Linear(1, 1, bias=False), - ), - "linear": nn.Sequential(nn.Linear(1, 1, bias=True)), - "relu": nn.Sequential( - nn.ReLU(inplace=False), - ), - "linear_relu": nn.Sequential( - nn.Linear(1, 1, bias=True), - nn.ReLU(inplace=False), - ), - "logistic": nn.Sequential( - nn.Sigmoid(), - ), - "linear_logistic": nn.Sequential( - nn.Linear(1, 1, bias=True), - nn.Sigmoid(), - ), - "exp": nn.Sequential( - Exponential(), - ), - "linear_exp": nn.Sequential( - nn.Linear(1, 1, bias=True), - Exponential(), - ), - "cos": nn.Sequential( - Cosine(), - ), - "linear_cos": nn.Sequential( - nn.Linear(1, 1, bias=True), - Cosine(), - ), - "sin": nn.Sequential( - Sine(), - ), - "linear_sin": nn.Sequential( - nn.Linear(1, 1, bias=True), - Sine(), - ), - "tanh": nn.Sequential( - Tangens_Hyperbolicus(), - ), - "linear_tanh": nn.Sequential( - nn.Linear(1, 1, bias=True), - Tangens_Hyperbolicus(), - ), - "reciprocal": nn.Sequential( - MultInverse(), - ), - "linear_reciprocal": nn.Sequential( - nn.Linear(1, 1, bias=False), - MultInverse(), - ), - "ln": nn.Sequential( - NatLogarithm(), - ), - "linear_ln": nn.Sequential( - nn.Linear(1, 1, bias=False), - NatLogarithm(), - ), - "softplus": nn.Sequential( - Softplus(), - ), - "linear_softplus": nn.Sequential( - nn.Linear(1, 1, bias=False), - Softplus(), - ), - "softminus": nn.Sequential( - Softminus(), - ), - "linear_softminus": nn.Sequential( - nn.Linear(1, 1, bias=False), - Softminus(), - ), -} + + +def operation_factory(name): + + if name == "none": + return Zero(1) + elif name == "add": + return nn.Sequential(Identity()) + elif name == "subtract": + return nn.Sequential(NegIdentity()) + elif name == "mult": + return nn.Sequential( + nn.Linear(1, 1, bias=False), + ) + elif name == "linear": + return nn.Sequential(nn.Linear(1, 1, bias=True)) + elif name == "relu": + return nn.Sequential( + nn.ReLU(inplace=False), + ) + elif name == "linear_relu": + return nn.Sequential( + nn.Linear(1, 1, bias=True), + nn.ReLU(inplace=False), + ) + elif name == "logistic": + return nn.Sequential( + nn.Sigmoid(), + ) + elif name == "linear_logistic": + return nn.Sequential( + nn.Linear(1, 1, bias=True), + nn.Sigmoid(), + ) + elif name == "exp": + return nn.Sequential( + Exponential(), + ) + elif name == "linear_exp": + return nn.Sequential( + nn.Linear(1, 1, bias=True), + Exponential(), + ) + elif name == "cos": + return nn.Sequential( + Cosine(), + ) + elif name == "linear_cos": + return nn.Sequential( + nn.Linear(1, 1, bias=True), + Cosine(), + ) + elif name == "sin": + return nn.Sequential( + Sine(), + ) + elif name == "linear_sin": + return nn.Sequential( + nn.Linear(1, 1, bias=True), + Sine(), + ) + elif name == "tanh": + return nn.Sequential( + Tangens_Hyperbolicus(), + ) + elif name == "linear_tanh": + return nn.Sequential( + nn.Linear(1, 1, bias=True), + Tangens_Hyperbolicus(), + ) + elif name == "reciprocal": + return nn.Sequential( + MultInverse(), + ) + elif name == "linear_reciprocal": + return nn.Sequential( + nn.Linear(1, 1, bias=False), + MultInverse(), + ) + elif name == "ln": + return nn.Sequential( + NatLogarithm(), + ) + elif name == "linear_ln": + return nn.Sequential( + nn.Linear(1, 1, bias=False), + NatLogarithm(), + ) + elif name == "softplus": + return nn.Sequential( + Softplus(), + ) + elif name == "linear_softplus": + return nn.Sequential( + nn.Linear(1, 1, bias=False), + Softplus(), + ) + elif name == "softminus": + return nn.Sequential( + Softminus(), + ) + elif name == "linear_softminus": + return nn.Sequential( + nn.Linear(1, 1, bias=False), + Softminus(), + ) + else: + raise NotImplementedError(f"operation {name=} it not implemented") + # this is the list of primitives actually used, # and it should be a set of names contained in the OPS dictionary @@ -632,4 +662,4 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # make sure that every primitive is in the OPS dictionary for name in PRIMITIVES: - assert name in OPS + assert operation_factory(name) is not None From 361eb8b2b725192d829d87b3eca4672c3717d272 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 16:20:23 -0500 Subject: [PATCH 33/50] added check that all models must have the same prediction method --- autora/experimentalist/sampler/model_disagreement.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index 1e6a80955..e6674c70e 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -37,10 +37,11 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): model_A_predict = getattr(model_A, "predict_proba", None) if callable(model_A_predict) is False: model_A_predict = getattr(model_A, "predict", None) - - model_B_predict = getattr(model_B, "predict_proba", None) - if callable(model_B_predict) is False: model_B_predict = getattr(model_B, "predict", None) + else: + model_B_predict = getattr(model_B, "predict_proba", None) + if callable(model_B_predict) is False: + raise Exception("Models must have the same prediction method.") if model_A_predict is None or model_B_predict is None: raise Exception("Model must have `predict` or `predict_proba` method.") From 28fc04052bfc423f4875e2205faef729a3348aa5 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 16:22:47 -0500 Subject: [PATCH 34/50] using itertools instead of double for loop --- .../sampler/model_disagreement.py | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index e6674c70e..3efd54ef6 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -1,3 +1,4 @@ +import itertools from typing import Iterable, List import numpy as np @@ -26,40 +27,35 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): model_disagreement = list() # collect diagreements for each model apir - for idx_A, model_A in enumerate(models): - for idx_B, model_B in enumerate(models): - - # don't compare the model with itself - if idx_A <= idx_B: - continue - - # determine the prediction method - model_A_predict = getattr(model_A, "predict_proba", None) - if callable(model_A_predict) is False: - model_A_predict = getattr(model_A, "predict", None) - model_B_predict = getattr(model_B, "predict", None) - else: - model_B_predict = getattr(model_B, "predict_proba", None) - if callable(model_B_predict) is False: - raise Exception("Models must have the same prediction method.") - - if model_A_predict is None or model_B_predict is None: - raise Exception("Model must have `predict` or `predict_proba` method.") - - # get predictions from both models - Y_A = model_A_predict(X_predict) - Y_B = model_B_predict(X_predict) - - if Y_A.shape != Y_B.shape: - raise Exception("Models must have same output shape.") - - # determine the disagreement between the two models in terms of mean-squared error - if len(Y_A.shape) == 1: - disagreement = (Y_A - Y_B) ** 2 - else: - disagreement = np.mean((Y_A - Y_B) ** 2, axis=1) - - model_disagreement.append(disagreement) + for model_A, model_B in itertools.combinations(models, 2): + + # determine the prediction method + model_A_predict = getattr(model_A, "predict_proba", None) + if callable(model_A_predict) is False: + model_A_predict = getattr(model_A, "predict", None) + model_B_predict = getattr(model_B, "predict", None) + else: + model_B_predict = getattr(model_B, "predict_proba", None) + if callable(model_B_predict) is False: + raise Exception("Models must have the same prediction method.") + + if model_A_predict is None or model_B_predict is None: + raise Exception("Model must have `predict` or `predict_proba` method.") + + # get predictions from both models + Y_A = model_A_predict(X_predict) + Y_B = model_B_predict(X_predict) + + if Y_A.shape != Y_B.shape: + raise Exception("Models must have same output shape.") + + # determine the disagreement between the two models in terms of mean-squared error + if len(Y_A.shape) == 1: + disagreement = (Y_A - Y_B) ** 2 + else: + disagreement = np.mean((Y_A - Y_B) ** 2, axis=1) + + model_disagreement.append(disagreement) if len(model_disagreement) == 0: raise Exception("No models to compare.") From f0c255a0e69b78b27e2fa2667814e0042dcb0ab8 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 16:24:24 -0500 Subject: [PATCH 35/50] using all lower case variable naming --- .../sampler/model_disagreement.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index 3efd54ef6..4c149e13b 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -27,33 +27,33 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): model_disagreement = list() # collect diagreements for each model apir - for model_A, model_B in itertools.combinations(models, 2): + for model_a, model_b in itertools.combinations(models, 2): # determine the prediction method - model_A_predict = getattr(model_A, "predict_proba", None) - if callable(model_A_predict) is False: - model_A_predict = getattr(model_A, "predict", None) - model_B_predict = getattr(model_B, "predict", None) + model_a_predict = getattr(model_a, "predict_proba", None) + if callable(model_a_predict) is False: + model_a_predict = getattr(model_a, "predict", None) + model_b_predict = getattr(model_b, "predict", None) else: - model_B_predict = getattr(model_B, "predict_proba", None) - if callable(model_B_predict) is False: + model_b_predict = getattr(model_b, "predict_proba", None) + if callable(model_b_predict) is False: raise Exception("Models must have the same prediction method.") - if model_A_predict is None or model_B_predict is None: + if model_a_predict is None or model_b_predict is None: raise Exception("Model must have `predict` or `predict_proba` method.") # get predictions from both models - Y_A = model_A_predict(X_predict) - Y_B = model_B_predict(X_predict) + y_a = model_a_predict(X_predict) + y_b = model_b_predict(X_predict) - if Y_A.shape != Y_B.shape: + if y_a.shape != y_b.shape: raise Exception("Models must have same output shape.") # determine the disagreement between the two models in terms of mean-squared error - if len(Y_A.shape) == 1: - disagreement = (Y_A - Y_B) ** 2 + if len(y_a.shape) == 1: + disagreement = (y_a - y_b) ** 2 else: - disagreement = np.mean((Y_A - Y_B) ** 2, axis=1) + disagreement = np.mean((y_a - y_b) ** 2, axis=1) model_disagreement.append(disagreement) From 3c30bd11f51216b4e70e740e726a3a5fcb090f6a Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 16:35:32 -0500 Subject: [PATCH 36/50] update typing for nclasses --- autora/experimentalist/sampler/poppernet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/poppernet.py b/autora/experimentalist/sampler/poppernet.py index dc39ce744..9508deaad 100644 --- a/autora/experimentalist/sampler/poppernet.py +++ b/autora/experimentalist/sampler/poppernet.py @@ -1,4 +1,4 @@ -from typing import Iterable, Tuple, cast +from typing import Iterable, Optional, Tuple, cast import numpy as np import torch @@ -312,7 +312,7 @@ def freeze_weights(self): param.requires_grad = False -def class_to_onehot(y: np.array, n_classes: int = None): +def class_to_onehot(y: np.array, n_classes: Optional[int] = None): """Converts a class vector (integers) to binary class matrix. E.g. for use with categorical_crossentropy. From a50b2bcb3bbc0ab03b66f5df0475b617a544aa62 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 16:46:00 -0500 Subject: [PATCH 37/50] removed comment --- tests/test_poppernet_sampler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py index 532e3fc83..667a91777 100644 --- a/tests/test_poppernet_sampler.py +++ b/tests/test_poppernet_sampler.py @@ -59,7 +59,6 @@ def classification_data_to_test(): @pytest.fixture def regression_data_to_test(): - # data = np.linspace(0, 2 * np.pi, 5) data = [-10, 0, 1.5, 3, 4.5, 6, 10] return data From 5e5d9f5559c487af9635f076fa760147207775de Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 16:54:16 -0500 Subject: [PATCH 38/50] test: correct import statement --- tests/test_poppernet_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py index 667a91777..7f2ca8c44 100644 --- a/tests/test_poppernet_sampler.py +++ b/tests/test_poppernet_sampler.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from experimentalist.pipeline import Pipeline from sklearn.linear_model import LinearRegression, LogisticRegression +from autora.experimentalist.pipeline import Pipeline from autora.experimentalist.sampler.poppernet import ( nearest_values_sampler, poppernet_pooler, From e65ca5bbdf62f283f1d150ecb812a6841b8ac686 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 17:52:53 -0500 Subject: [PATCH 39/50] Update autora/experimentalist/filter.py Co-authored-by: Sebastian Musslick --- autora/experimentalist/filter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index f67bc150e..c631d2db7 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -35,7 +35,8 @@ def train_test_filter( >>> train_filter, test_filter = train_test_filter(train_p=0.6, seed=180) The `train_filter` generates a sequence of ~60% of the input list – - in this case, 15 of 20 datapoints. + in this case, 15 of 20 datapoints. Note that the correct split would be 12 of 20 data points. Again, + for data with bounded length it is advisable to use scikit-learn `train_test_split` instead. >>> list(train_filter(range(20))) [0, 2, 3, 4, 5, 6, 9, 10, 11, 12, 15, 16, 17, 18, 19] From 6129c6e2e295d7075b0615bed295a5c081b44e4f Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 18:01:10 -0500 Subject: [PATCH 40/50] removed comment --- tests/test_poppernet_sampler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_poppernet_sampler.py b/tests/test_poppernet_sampler.py index 7f2ca8c44..7f5c0a5bd 100644 --- a/tests/test_poppernet_sampler.py +++ b/tests/test_poppernet_sampler.py @@ -69,7 +69,6 @@ def test_poppernet_classification(synthetic_logr_model, classification_data_to_t X_train, Y_train = get_xor_data() X = classification_data_to_test model = synthetic_logr_model - # specify meta data # Specify independent variables iv1 = IV( From 3d4a56359ad8061f224696ce9b0b71a513dfad11 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 18:12:45 -0500 Subject: [PATCH 41/50] test: fix testcase to give a NotImplement4edError --- tests/test_sklearn_darts.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_sklearn_darts.py b/tests/test_sklearn_darts.py index 86e912ded..e62df2a27 100644 --- a/tests/test_sklearn_darts.py +++ b/tests/test_sklearn_darts.py @@ -4,8 +4,6 @@ import pandas as pd import pytest import torch -from sklearn.model_selection import GridSearchCV, train_test_split - from autora.skl.darts import ( PRIMITIVES, DARTSExecutionMonitor, @@ -13,6 +11,7 @@ DARTSType, ValueType, ) +from sklearn.model_selection import GridSearchCV, train_test_split def generate_noisy_constant_data( @@ -129,8 +128,10 @@ def test_primitive_selection(): DARTSRegressor(primitives=["add", "subtract", "none"], **kwargs).fit(X, y) DARTSRegressor(primitives=PRIMITIVES, **kwargs).fit(X, y) - with pytest.raises(KeyError): - KeyError, DARTSRegressor(primitives=["doesnt_exist"], **kwargs).fit(X, y) + with pytest.raises(NotImplementedError): + NotImplementedError, DARTSRegressor(primitives=["doesnt_exist"], **kwargs).fit( + X, y + ) def test_fit_with_fixed_architecture(): From 79be0d05c7ddbd127dd0635cc504abe69f045eaa Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 18:15:06 -0500 Subject: [PATCH 42/50] docs: update docstring --- autora/experimentalist/filter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/filter.py b/autora/experimentalist/filter.py index c631d2db7..58a9139cd 100644 --- a/autora/experimentalist/filter.py +++ b/autora/experimentalist/filter.py @@ -35,8 +35,10 @@ def train_test_filter( >>> train_filter, test_filter = train_test_filter(train_p=0.6, seed=180) The `train_filter` generates a sequence of ~60% of the input list – - in this case, 15 of 20 datapoints. Note that the correct split would be 12 of 20 data points. Again, - for data with bounded length it is advisable to use scikit-learn `train_test_split` instead. + in this case, 15 of 20 datapoints. + Note that the correct split would be 12 of 20 data points. + Again, for data with bounded length it is advisable + to use scikit-learn `train_test_split` instead. >>> list(train_filter(range(20))) [0, 2, 3, 4, 5, 6, 9, 10, 11, 12, 15, 16, 17, 18, 19] From 8002e690876b9e7822e7c474fcd9a81f38ea093b Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 18:21:45 -0500 Subject: [PATCH 43/50] chore: reorder imports --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc1295a22..39ccaab09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: args: - "--profile=black" - "--filter-files" + - "--project=autora" - repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: From 77c7ef37d290c1f96f16bf61d074b0d8d9ef920f Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 18:21:48 -0500 Subject: [PATCH 44/50] chore: reorder imports --- tests/test_sklearn_darts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_sklearn_darts.py b/tests/test_sklearn_darts.py index e62df2a27..03ac817c7 100644 --- a/tests/test_sklearn_darts.py +++ b/tests/test_sklearn_darts.py @@ -4,6 +4,8 @@ import pandas as pd import pytest import torch +from sklearn.model_selection import GridSearchCV, train_test_split + from autora.skl.darts import ( PRIMITIVES, DARTSExecutionMonitor, @@ -11,7 +13,6 @@ DARTSType, ValueType, ) -from sklearn.model_selection import GridSearchCV, train_test_split def generate_noisy_constant_data( From e8d9b3a207d95816d17d2ab65a7f7f6567479bb2 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Tue, 6 Dec 2022 18:21:45 -0500 Subject: [PATCH 45/50] chore: reorder imports --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc1295a22..39ccaab09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: args: - "--profile=black" - "--filter-files" + - "--project=autora" - repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: From 14447715b7ead4cf1f77002e5c6fde52ed4ad307 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 18:47:55 -0500 Subject: [PATCH 46/50] Update autora/experimentalist/sampler/model_disagreement.py Co-authored-by: John Gerrard Holland --- .../sampler/model_disagreement.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index 4c149e13b..9d6a64be8 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -30,17 +30,14 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): for model_a, model_b in itertools.combinations(models, 2): # determine the prediction method - model_a_predict = getattr(model_a, "predict_proba", None) - if callable(model_a_predict) is False: - model_a_predict = getattr(model_a, "predict", None) - model_b_predict = getattr(model_b, "predict", None) + if hasattr(model_a, "predict_proba") and hasattr(model_b, "predict_proba"): + model_a_predict = model_a.predict_proba + model_b_predict = model_b.predict_proba + elif hasattr(model_a, "predict") and hasattr(model_b, "predict"): + model_a_predict = model_a.predict + model_b_predict = model_b.predict else: - model_b_predict = getattr(model_b, "predict_proba", None) - if callable(model_b_predict) is False: - raise Exception("Models must have the same prediction method.") - - if model_a_predict is None or model_b_predict is None: - raise Exception("Model must have `predict` or `predict_proba` method.") + raise AttributeError("Models must both have `predict_proba` or `predict` method.") # get predictions from both models y_a = model_a_predict(X_predict) From 71fcd474f4111815921f8bbaf819a67dc04711bb Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 18:49:05 -0500 Subject: [PATCH 47/50] Update autora/experimentalist/sampler/model_disagreement.py Co-authored-by: John Gerrard Holland --- autora/experimentalist/sampler/model_disagreement.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index 9d6a64be8..a1c113559 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -43,8 +43,7 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): y_a = model_a_predict(X_predict) y_b = model_b_predict(X_predict) - if y_a.shape != y_b.shape: - raise Exception("Models must have same output shape.") + assert y_a.shape == y_b.shape, "Models must have same output shape." # determine the disagreement between the two models in terms of mean-squared error if len(y_a.shape) == 1: From 212919305e6c4dc2e6c686441c41c22b63e574d7 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 18:50:00 -0500 Subject: [PATCH 48/50] autora/experimentalist/sampler/model_disagreement.py --- autora/experimentalist/sampler/model_disagreement.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index a1c113559..49291bf50 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -26,7 +26,7 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): model_disagreement = list() - # collect diagreements for each model apir + # collect diagreements for each model pair for model_a, model_b in itertools.combinations(models, 2): # determine the prediction method @@ -37,7 +37,9 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): model_a_predict = model_a.predict model_b_predict = model_b.predict else: - raise AttributeError("Models must both have `predict_proba` or `predict` method.") + raise AttributeError( + "Models must both have `predict_proba` or `predict` method." + ) # get predictions from both models y_a = model_a_predict(X_predict) From 31def0a12c2ca46e155b7a3e367ea59f45feec72 Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 18:51:39 -0500 Subject: [PATCH 49/50] Update autora/experimentalist/sampler/model_disagreement.py Co-authored-by: John Gerrard Holland --- autora/experimentalist/sampler/model_disagreement.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index 49291bf50..bd15883d3 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -55,8 +55,7 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): model_disagreement.append(disagreement) - if len(model_disagreement) == 0: - raise Exception("No models to compare.") + assert len(model_disagreement) >= 1, "No disagreements to compare." # sum up all model disagreements summed_disagreement = np.sum(model_disagreement, axis=0) From aed4e86646b6d60bfdad44c5a466d274cca16aea Mon Sep 17 00:00:00 2001 From: Sebastian Musslick Date: Tue, 6 Dec 2022 18:52:48 -0500 Subject: [PATCH 50/50] adjusted comment --- autora/experimentalist/sampler/model_disagreement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autora/experimentalist/sampler/model_disagreement.py b/autora/experimentalist/sampler/model_disagreement.py index bd15883d3..20a9b805f 100644 --- a/autora/experimentalist/sampler/model_disagreement.py +++ b/autora/experimentalist/sampler/model_disagreement.py @@ -60,7 +60,7 @@ def model_disagreement_sampler(X: np.array, models: List, num_samples: int = 1): # sum up all model disagreements summed_disagreement = np.sum(model_disagreement, axis=0) - # sort the summed disagreements + # sort the summed disagreements and select the top n idx = (-summed_disagreement).argsort()[:num_samples] return X[idx]