From 570201057ced108ec05e713f81304bf064391710 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 2 May 2023 18:42:24 -0700 Subject: [PATCH 1/6] Fix calculation of best observed value --- xopt/generators/bayesian/expected_improvement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xopt/generators/bayesian/expected_improvement.py b/xopt/generators/bayesian/expected_improvement.py index baee386a..1aac2abf 100644 --- a/xopt/generators/bayesian/expected_improvement.py +++ b/xopt/generators/bayesian/expected_improvement.py @@ -50,7 +50,7 @@ def _get_acquisition(self, model): ].dropna() objective_data = self.vocs.objective_data(valid_data, "") - best_f = torch.tensor(objective_data.max(), **self._tkwargs) + best_f = -torch.tensor(objective_data.min(), **self._tkwargs) qEI = qExpectedImprovement( model, From 3028b6164b47d729647b78b6863a13b241b15e9e Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Tue, 2 May 2023 18:43:33 -0700 Subject: [PATCH 2/6] Add test for acquisition accuracy --- .../bayesian/test_expected_improvement.py | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/generators/bayesian/test_expected_improvement.py b/tests/generators/bayesian/test_expected_improvement.py index 807f14c1..875bcf96 100644 --- a/tests/generators/bayesian/test_expected_improvement.py +++ b/tests/generators/bayesian/test_expected_improvement.py @@ -1,14 +1,16 @@ +import torch +import numpy as np from copy import deepcopy - import pytest +from botorch.optim import optimize_acqf +from botorch.acquisition import ExpectedImprovement from xopt.base import Xopt - from xopt.evaluator import Evaluator from xopt.generators.bayesian.expected_improvement import ExpectedImprovementGenerator from xopt.generators.bayesian.upper_confidence_bound import UCBOptions - from xopt.resources.testing import TEST_VOCS_BASE, TEST_VOCS_DATA, xtest_callable +from xopt.resources.test_functions.sinusoid_1d import sinusoid_vocs, evaluate_sinusoid class TestExpectedImprovement: @@ -82,3 +84,48 @@ def test_in_xopt_w_proximal(self): # now use bayes opt for _ in range(1): xopt.step() + + def test_acquisition_accuracy(self): + # fix random seeds for BoTorch + torch.manual_seed(0) + np.random.seed(0) + + vocs = sinusoid_vocs + vocs.constraints = {} + for objective in ["MINIMIZE", "MAXIMIZE"]: + vocs.objectives["y1"] = objective + evaluator = Evaluator(function=evaluate_sinusoid) + generator = ExpectedImprovementGenerator(vocs) + X = Xopt(evaluator=evaluator, generator=generator, vocs=vocs) + X.step() + + distance = 0.0 + for i in range(3): + model = X.generator.train_model() + + # analytical acquisition + if objective == "MAXIMIZE": + maximize = True + best_f = torch.tensor(X.data["y1"].values).max() + else: + maximize = False + best_f = torch.tensor(X.data["y1"].values).min() + acq_analytical = ExpectedImprovement(model, best_f=best_f, + maximize=maximize) + candidate_analytical, _ = optimize_acqf( + acq_function=acq_analytical, + bounds=torch.tensor(vocs.bounds), + q=1, + num_restarts=20, + raw_samples=100 + ) + + # xopt step + X.step() + + # calculate distance from analytical candidate + distance += torch.abs( + X.data["x1"].values[-1] - candidate_analytical.squeeze()) + + # distance should be small + assert distance < 0.1 From 7e58f0719ce6d9b2777227fff316ee6be0900891 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Wed, 3 May 2023 12:19:27 -0700 Subject: [PATCH 3/6] Improve test for acquisition accuracy --- .../bayesian/test_expected_improvement.py | 79 ++++++++----------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/tests/generators/bayesian/test_expected_improvement.py b/tests/generators/bayesian/test_expected_improvement.py index 875bcf96..d5fe153c 100644 --- a/tests/generators/bayesian/test_expected_improvement.py +++ b/tests/generators/bayesian/test_expected_improvement.py @@ -1,16 +1,15 @@ import torch -import numpy as np +import pandas as pd from copy import deepcopy import pytest -from botorch.optim import optimize_acqf from botorch.acquisition import ExpectedImprovement from xopt.base import Xopt +from xopt.vocs import VOCS, ObjectiveEnum from xopt.evaluator import Evaluator from xopt.generators.bayesian.expected_improvement import ExpectedImprovementGenerator from xopt.generators.bayesian.upper_confidence_bound import UCBOptions from xopt.resources.testing import TEST_VOCS_BASE, TEST_VOCS_DATA, xtest_callable -from xopt.resources.test_functions.sinusoid_1d import sinusoid_vocs, evaluate_sinusoid class TestExpectedImprovement: @@ -86,46 +85,36 @@ def test_in_xopt_w_proximal(self): xopt.step() def test_acquisition_accuracy(self): - # fix random seeds for BoTorch - torch.manual_seed(0) - np.random.seed(0) - - vocs = sinusoid_vocs - vocs.constraints = {} - for objective in ["MINIMIZE", "MAXIMIZE"]: - vocs.objectives["y1"] = objective - evaluator = Evaluator(function=evaluate_sinusoid) + train_x = torch.tensor([0.01, 0.3, 0.6, 0.99]).double() + train_y = torch.sin(2 * torch.pi * train_x) + train_data = pd.DataFrame( + {"x1": train_x.numpy(), "y1": train_y.numpy()}) + test_x = torch.linspace(0.0, 1.0, 1000) + + for objective in ObjectiveEnum: + vocs = VOCS(**{"variables": {"x1": [0.0, 1.0]}, + "objectives": {"y1": objective}}) generator = ExpectedImprovementGenerator(vocs) - X = Xopt(evaluator=evaluator, generator=generator, vocs=vocs) - X.step() - - distance = 0.0 - for i in range(3): - model = X.generator.train_model() - - # analytical acquisition - if objective == "MAXIMIZE": - maximize = True - best_f = torch.tensor(X.data["y1"].values).max() - else: - maximize = False - best_f = torch.tensor(X.data["y1"].values).min() - acq_analytical = ExpectedImprovement(model, best_f=best_f, - maximize=maximize) - candidate_analytical, _ = optimize_acqf( - acq_function=acq_analytical, - bounds=torch.tensor(vocs.bounds), - q=1, - num_restarts=20, - raw_samples=100 - ) - - # xopt step - X.step() - - # calculate distance from analytical candidate - distance += torch.abs( - X.data["x1"].values[-1] - candidate_analytical.squeeze()) - - # distance should be small - assert distance < 0.1 + generator.add_data(train_data) + model = generator.train_model().models[0] + + # xopt acquisition function + acq = generator.get_acquisition(model) + + # analytical acquisition function + if objective == "MAXIMIZE": + an_acq = ExpectedImprovement(model, best_f=train_y.max(), + maximize=True) + else: + an_acq = ExpectedImprovement(model, best_f=train_y.min(), + maximize=False) + + # compare candidates (maximum in test data) + with torch.no_grad(): + acq_v = acq(test_x.reshape(-1, 1, 1)) + candidate = test_x[torch.argmax(acq_v)] + an_acq_v = an_acq(test_x.reshape(-1, 1, 1)) + an_candidate = test_x[torch.argmax(an_acq_v)] + + # difference should be small + assert torch.abs(an_candidate - candidate) < 0.01 From 2460dc6935dbd99a3fdbc5a71f1a29d5794bcf28 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Wed, 3 May 2023 12:51:10 -0700 Subject: [PATCH 4/6] Ignore NaNs independently --- xopt/generators/bayesian/expected_improvement.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/xopt/generators/bayesian/expected_improvement.py b/xopt/generators/bayesian/expected_improvement.py index 1aac2abf..a5471fa4 100644 --- a/xopt/generators/bayesian/expected_improvement.py +++ b/xopt/generators/bayesian/expected_improvement.py @@ -45,11 +45,7 @@ def default_options() -> BayesianOptions: return BayesianOptions() def _get_acquisition(self, model): - valid_data = self.data[ - pd.unique(self.vocs.variable_names + self.vocs.output_names) - ].dropna() - objective_data = self.vocs.objective_data(valid_data, "") - + objective_data = self.vocs.objective_data(self.data, "").dropna() best_f = -torch.tensor(objective_data.min(), **self._tkwargs) qEI = qExpectedImprovement( From f87987e0cc119335835c76c03adf9b75522bc7e3 Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Wed, 3 May 2023 12:56:07 -0700 Subject: [PATCH 5/6] Delete unused import --- xopt/generators/bayesian/expected_improvement.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xopt/generators/bayesian/expected_improvement.py b/xopt/generators/bayesian/expected_improvement.py index a5471fa4..08340b11 100644 --- a/xopt/generators/bayesian/expected_improvement.py +++ b/xopt/generators/bayesian/expected_improvement.py @@ -1,4 +1,3 @@ -import pandas as pd import torch from botorch.acquisition import qExpectedImprovement From 7fbadde757314e9f6f60f461d591d820e09583ed Mon Sep 17 00:00:00 2001 From: Tobias Boltz Date: Wed, 3 May 2023 14:01:30 -0700 Subject: [PATCH 6/6] Revert to base acquisition if there are no constraints --- .../custom_botorch/constrained_acqusition.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/xopt/generators/bayesian/custom_botorch/constrained_acqusition.py b/xopt/generators/bayesian/custom_botorch/constrained_acqusition.py index 805716d0..8a051d52 100644 --- a/xopt/generators/bayesian/custom_botorch/constrained_acqusition.py +++ b/xopt/generators/bayesian/custom_botorch/constrained_acqusition.py @@ -72,12 +72,17 @@ def __init__( @concatenate_pending_points @t_batch_mode_transform() def forward(self, X: Tensor) -> Tensor: - posterior = self.model.posterior( - X=X, posterior_transform=self.posterior_transform - ) - samples = self.get_posterior_samples(posterior) - obj = self.objective(samples, X=X) + if self.objective.constraints: + posterior = self.model.posterior( + X=X, posterior_transform=self.posterior_transform + ) + samples = self.get_posterior_samples(posterior) + obj = self.objective(samples, X=X) - # multiply the output of the base acquisition function by the feasibility - base_val = torch.nn.functional.softplus(self.base_acqusition(X), beta=10) - return base_val * obj.max(dim=-1)[0].mean(dim=0) + # multiply the output of the base acquisition function by + # the feasibility + base_val = torch.nn.functional.softplus( + self.base_acqusition(X), beta=10) + return base_val * obj.max(dim=-1)[0].mean(dim=0) + else: + return self.base_acqusition(X)