From 59ff383d832904789e4ccf63d5bb99376978e7c9 Mon Sep 17 00:00:00 2001 From: Russ Tedrake Date: Fri, 30 Jun 2023 18:19:16 -0400 Subject: [PATCH] Never destroy local Gurobi licenses Iff the contents of the file identified in GRB_LICENSE_FILE contains the magic keywork HOSTID, then any license that is obtained will never be destroyed. Resolves #19657. A new implementation and unit test strategy --- solvers/BUILD.bazel | 22 +++++++++ solvers/gurobi_solver.cc | 45 +++++++++++++++++++ solvers/gurobi_solver.h | 16 ++++--- .../gurobi_solver_license_retention_test.py | 43 ++++++++++++++++++ ...bi_solver_license_retention_test_helper.cc | 11 +++++ 5 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 solvers/test/gurobi_solver_license_retention_test.py create mode 100644 solvers/test/gurobi_solver_license_retention_test_helper.cc diff --git a/solvers/BUILD.bazel b/solvers/BUILD.bazel index 6c2aa243b943..1eade9bf4240 100644 --- a/solvers/BUILD.bazel +++ b/solvers/BUILD.bazel @@ -6,6 +6,10 @@ load( "drake_cc_package_library", "drake_cc_test", ) +load( + "@drake//tools/skylark:drake_py.bzl", + "drake_py_unittest", +) load( ":defs.bzl", "drake_cc_optional_googletest", @@ -1249,6 +1253,24 @@ drake_cc_googletest( ], ) +drake_py_unittest( + name = "gurobi_solver_license_retention_test", + data = [ + ":gurobi_solver_license_retention_test_helper", + ], + tags = gurobi_test_tags(), +) + +drake_cc_binary( + name = "gurobi_solver_license_retention_test_helper", + testonly = True, + srcs = ["test/gurobi_solver_license_retention_test_helper.cc"], + visibility = ["//visibility:private"], + deps = [ + ":gurobi_solver", + ], +) + drake_cc_googletest( name = "integer_optimization_util_test", deps = [ diff --git a/solvers/gurobi_solver.cc b/solvers/gurobi_solver.cc index 5247d783674f..ea9570f4934f 100644 --- a/solvers/gurobi_solver.cc +++ b/solvers/gurobi_solver.cc @@ -3,9 +3,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -1039,6 +1041,14 @@ class GurobiSolver::License { "Could not locate Gurobi license key file because GRB_LICENSE_FILE " "environment variable was not set."); } + if (const char* filename = std::getenv("GRB_LICENSE_FILE")) { + // For unit testing, we employ a hack to keep env_ uninitialized so that + // we don't need a valid license file. + if (std::string_view{filename}.find("DRAKE_UNIT_TEST_NO_LICENSE") != + std::string_view::npos) { + return; + } + } const int num_tries = 3; int grb_load_env_error = 1; for (int i = 0; grb_load_env_error && i < num_tries; ++i) { @@ -1066,7 +1076,42 @@ class GurobiSolver::License { GRBenv* env_ = nullptr; }; +namespace { +bool IsGrbLicenseFileLocalHost() { + // We use the existence of the string HOSTID in the license file as + // confirmation that the license is associated with the local host. + const char* grb_license_file = std::getenv("GRB_LICENSE_FILE"); + if (grb_license_file == nullptr) { + return false; + } + std::ifstream stream{grb_license_file}; + const std::string contents{std::istreambuf_iterator{stream}, + std::istreambuf_iterator{}}; + if (stream.fail()) { + return false; + } + return contents.find("HOSTID") != std::string::npos; +} +} // namespace + std::shared_ptr GurobiSolver::AcquireLicense() { + // Gurobi recommends acquiring the license only once per program to avoid + // overhead from acquiring the license (and console spew for academic license + // users; see #19657). However, if users are using a shared network license + // from a limited pool, then we risk them checking out the license and not + // giving it back (e.g., if they are working in a jupyter notebook). As a + // compromise, we extend license beyond the lifetime of the GurobiSolver iff + // we can confirm that the license is associated with the local host. + // + // The first time the anyone calls GurobiSolver::AcquireLicense, we check + // whether the license is local. If yes, the local_host_holder keeps the + // license's use_count lower bounded to 1. If no, the local_hold_holder is + // null and the usual GetScopedSingleton workflow applies. + static never_destroyed> local_host_holder{[]() { + return IsGrbLicenseFileLocalHost() + ? GetScopedSingleton() + : nullptr; + }()}; return GetScopedSingleton(); } diff --git a/solvers/gurobi_solver.h b/solvers/gurobi_solver.h index a9f1bcc4b9f2..2cffbd28ac62 100644 --- a/solvers/gurobi_solver.h +++ b/solvers/gurobi_solver.h @@ -163,13 +163,15 @@ class GurobiSolver final : public SolverBase { /** * This acquires a Gurobi license environment shared among all GurobiSolver - * instances; the environment will stay valid as long as at least one - * shared_ptr returned by this function is alive. - * Call this ONLY if you must use different MathematicalProgram - * instances at different instances in time, and repeatedly acquiring the - * license is costly (e.g., requires contacting a license server). - * @return A shared pointer to a license environment that will stay valid - * as long as any shared_ptr returned by this function is alive. If Gurobi + * instances. The environment will stay valid as long as at least one + * shared_ptr returned by this function is alive or if the license can be + * confirmed to be a local license then it will never be destroyed; see + * has_acquired_local_license(). Call this ONLY if you must use different + * MathematicalProgram instances at different instances in time, and + * repeatedly acquiring the license is costly (e.g., requires contacting a + * license server). + * @return A shared pointer to a license environment that will stay valid as + * long as any shared_ptr returned by this function is alive. If Gurobi is * not available in your build, this will return a null (empty) shared_ptr. * @throws std::exception if Gurobi is available but a license cannot be * obtained. diff --git a/solvers/test/gurobi_solver_license_retention_test.py b/solvers/test/gurobi_solver_license_retention_test.py new file mode 100644 index 000000000000..6353cdfd131f --- /dev/null +++ b/solvers/test/gurobi_solver_license_retention_test.py @@ -0,0 +1,43 @@ +import copy +import os +from pathlib import Path +import subprocess +import unittest + + +class TestGurobiSolverLicenseRetention(unittest.TestCase): + + def _subprocess_license_use_count(self, license_file_content): + """Sets GRB_LICENSE_FILE to a temp file with the given content, runs + our test helper, and then returns the license pointer use_count. + """ + # Create a dummy license file. Note that the license filename is magic. + # The License code in gurobi_solver.cc treats this filename specially. + tmpdir = Path(os.environ["TEST_TMPDIR"]) + license_file = tmpdir / "DRAKE_UNIT_TEST_NO_LICENSE.lic" + with open(license_file, "w", encoding="utf-8") as f: + f.write(license_file_content) + + # Override the built-in license file. + env = copy.copy(os.environ) + env["GRB_LICENSE_FILE"] = str(license_file) + + # Run the helper and return the pointer use_count. + output = subprocess.check_output( + ["solvers/gurobi_solver_license_retention_test_helper"]) + return int(output) + + def test_local_license(self): + """When the file named by GRB_LICENSE_FILE contains 'HOSTID', the + license object is held in two places: the test helper main(), and + a global variable within GurobiSolver::AcquireLicense. + """ + content = "HOSTID=foobar\n" + self.assertEqual(self._subprocess_license_use_count(content), 2) + + def test_nonlocal_license(self): + """When the file named by GRB_LICENSE_FILE doesn't contain 'HOSTID', + the license object is only held by main(), not any global variable. + """ + content = "TOKENSERVER=foobar.invalid.\n" + self.assertEqual(self._subprocess_license_use_count(content), 1) diff --git a/solvers/test/gurobi_solver_license_retention_test_helper.cc b/solvers/test/gurobi_solver_license_retention_test_helper.cc new file mode 100644 index 000000000000..1ee591767c70 --- /dev/null +++ b/solvers/test/gurobi_solver_license_retention_test_helper.cc @@ -0,0 +1,11 @@ +#include "drake/solvers/gurobi_solver.h" + +using drake::solvers::GurobiSolver; + +/* Acquire a license and report the overall use_count. */ +int main() { + std::shared_ptr license = + GurobiSolver::AcquireLicense(); + fmt::print("{}\n", license.use_count()); + return 0; +}