diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 85b99296bc6..1787bf8dafa 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -15,7 +15,7 @@ env: jobs: build: - name: ${{ matrix.os }} shrd=${{ matrix.cmake.shared }} java=${{ matrix.cmake.java }} dotnet=${{ matrix.cmake.dotnet }} python=${{ matrix.cmake.python }}-${{ matrix.cmake.python-version }} + name: ${{ matrix.cmake.config }} shrd=${{ matrix.cmake.shared }} java=${{ matrix.cmake.java }} dotnet=${{ matrix.cmake.dotnet }} python=${{ matrix.cmake.python }}-${{ matrix.cmake.python-version }} runs-on: ${{ matrix.os }} env: XPRESSDIR: ${{ github.workspace }}/xpressmp @@ -25,11 +25,12 @@ jobs: matrix: os: ["ubuntu-22.04"] cmake: [ - {shared: OFF, java: OFF, dotnet: OFF, python: OFF, python-version: "3.8", publish-cxx-or: ON}, - {shared: ON, java: ON, dotnet: ON, python: OFF, python-version: "3.8", publish-cxx-or: ON}, - {shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.8", publish-cxx-or: OFF}, - {shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.9", publish-cxx-or: OFF}, - {shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.10", publish-cxx-or: OFF}, + {config: "Release", shared: OFF, java: OFF, dotnet: OFF, python: OFF, python-version: "3.8", publish-cxx-or: ON}, + {config: "Release", shared: ON, java: ON, dotnet: ON, python: OFF, python-version: "3.8", publish-cxx-or: ON}, + {config: "Release", shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.8", publish-cxx-or: OFF}, + {config: "Debug", shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.8", publish-cxx-or: OFF}, + {config: "Release", shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.9", publish-cxx-or: OFF}, + {config: "Release", shared: ON, java: OFF, dotnet: OFF, python: ON, python-version: "3.10", publish-cxx-or: OFF}, ] steps: @@ -72,14 +73,14 @@ jobs: if: ${{ startsWith(matrix.os, 'ubuntu') }} uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ matrix.os }}-${{ matrix.cmake.shared }}-${{ matrix.cmake.dotnet }}-${{ matrix.cmake.java }}-${{ matrix.cmake.python }}-${{ matrix.cmake.python-version }} + key: ${{ matrix.cmake.config }}-${{ matrix.cmake.shared }}-${{ matrix.cmake.dotnet }}-${{ matrix.cmake.java }}-${{ matrix.cmake.python }}-${{ matrix.cmake.python-version }} - name: Check cmake run: cmake --version - name: Configure OR-Tools run: > cmake -S . -B build - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=${{ matrix.cmake.config }} -DBUILD_SHARED_LIBS=${{ matrix.cmake.shared }} -DBUILD_PYTHON=${{ matrix.cmake.python }} -DBUILD_JAVA=${{ matrix.cmake.java }} @@ -93,14 +94,14 @@ jobs: run: > cmake --build build - --config Release + --config ${{ matrix.cmake.config }} --target all install -j4 - name: run tests not xpress working-directory: ./build/ run: > ctest - -C Release + -C ${{ matrix.cmake.config }} --output-on-failure -E "xpress" @@ -109,7 +110,7 @@ jobs: run: > ctest -V - -C Release + -C ${{ matrix.cmake.config }} --output-on-failure -R "xpress" @@ -119,8 +120,10 @@ jobs: run: | SHARED=${{ matrix.cmake.shared }} [ $SHARED == "ON" ] && WITH_SHARED="_shared" || WITH_SHARED="_static" + CONFIG=${{ matrix.cmake.config }} + [ $CONFIG == "Debug" ] && WITH_DEBUG="_debug" || WITH_DEBUG="" OS="_${{ matrix.os }}" - APPENDIX="${OS}" + APPENDIX="${OS}${WITH_DEBUG}" echo "::set-output name=appendix::$APPENDIX" APPENDIX_WITH_SHARED="${OS}${WITH_SHARED}" echo "::set-output name=appendix_with_shared::$APPENDIX_WITH_SHARED" diff --git a/cmake/python.cmake b/cmake/python.cmake index 61755961876..d9362b56723 100644 --- a/cmake/python.cmake +++ b/cmake/python.cmake @@ -216,7 +216,7 @@ if(BUILD_TESTING) add_test( NAME python_${COMPONENT_NAME}_${TEST_NAME} - COMMAND ${VENV_Python3_EXECUTABLE} -m pytest ${FILE_NAME} + COMMAND ${VENV_Python3_EXECUTABLE} -m pytest ${FILE_NAME} -s WORKING_DIRECTORY ${VENV_DIR}) message(STATUS "Configuring test ${FILE_NAME} done") endfunction() diff --git a/ortools/linear_solver/java/XpressCallbackTest.java b/ortools/linear_solver/java/XpressCallbackTest.java new file mode 100644 index 00000000000..6b968af5965 --- /dev/null +++ b/ortools/linear_solver/java/XpressCallbackTest.java @@ -0,0 +1,135 @@ +package com.google.ortools.java; + +import com.google.ortools.Loader; +import com.google.ortools.linearsolver.*; +import java.lang.RuntimeException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +public class XpressCallbackTest { + // Numerical tolerance for checking primal, dual, objective values + // and other values. + private static final double NUM_TOLERANCE = 1e-5; + + static class CustomException extends RuntimeException { + public CustomException() { + } + + public CustomException(String msg) { + super(msg); + } + + public CustomException(Throwable throwable) { + super(throwable); + } + + public CustomException(String message, Throwable cause) { + super(message, cause); + } + } + + @BeforeEach + public void setUp() { + Loader.loadNativeLibraries(); + } + + static class MyMpCallback extends MPCallback { + private final MPSolver mpSolver; + private final boolean throwRuntimeException; + private int nSolutions = 0; + private List lastVarValues = null; + + public MyMpCallback(MPSolver mpSolver, boolean throwRuntimeException) { + super(false, false); + this.mpSolver = mpSolver; + this.throwRuntimeException = throwRuntimeException; + } + + @Override + public void runCallback(MPCallbackContext callback_context) { + if (throwRuntimeException) { + throw new CustomException("this is a test exception in a callback"); + } + if (!callback_context.event().equals(MPCallbackEvent.MIP_SOLUTION)) { + return; + } + nSolutions++; + if (!callback_context.canQueryVariableValues()) { + return; + } + lastVarValues = new ArrayList<>(); + for (MPVariable variable : mpSolver.variables()) { + lastVarValues.add(callback_context.variableValue(variable)); + } + } + } + + private MPSolver initSolver() { + MPSolver solver = MPSolver.createSolver("XPRESS_MIXED_INTEGER_PROGRAMMING"); + + int nVars = 30; + int maxTime = 30; + solver.objective().setMaximization(); + + Random rand = new Random(123); + for (int i = 0; i < nVars; i++) { + MPVariable x = solver.makeIntVar(-rand.nextDouble() * 200, rand.nextDouble() * 200, "x_" + i); + solver.objective().setCoefficient(x, rand.nextDouble() * 200 - 100); + if (i==0) { + continue; + } + double rand1 = -rand.nextDouble() * 2000; + double rand2 = rand.nextDouble() * 2000; + MPConstraint constraint = solver.makeConstraint(Math.min(rand1, rand2), Math.max(rand1, rand2)); + constraint.setCoefficient(x, rand.nextDouble() * 200 - 100); + for (int j = 0; j < i ; j++) { + constraint.setCoefficient(solver.variable(j), rand.nextDouble() * 200 - 100); + } + } + + solver.setSolverSpecificParametersAsString("PRESOLVE 0 MAXTIME " + maxTime); + solver.enableOutput(); + return solver; + } + + @Test + public void testXpressNewMipSolCallback() { + MPSolver solver = initSolver(); + if (solver == null) { + return; + } + + MyMpCallback cb = new MyMpCallback(solver, false); + solver.setCallback(cb); + + solver.solve(); + + + // This is a tough MIP, in 30 seconds XPRESS should have found at least 5 + // solutions (tested with XPRESS v9.0, may change in later versions) + assertTrue(cb.nSolutions > 5); + // Test that the last solution intercepted by callback is the same as the optimal one retained + for (int i = 0; i < solver.numVariables(); i++) { + assertEquals(solver.variable(i).solutionValue(), cb.lastVarValues.get(i), NUM_TOLERANCE); + } + } + + @Test + public void testCallbackThrowsException() { + // Test that when the callback throws an exception, it is caught by or-tools + MPSolver solver = initSolver(); + if (solver == null) { + return; + } + + MyMpCallback cb = new MyMpCallback(solver, true); + solver.setCallback(cb); + + assertDoesNotThrow(() -> solver.solve()); + } +} diff --git a/ortools/linear_solver/java/linear_solver.i b/ortools/linear_solver/java/linear_solver.i index 43dd1b78aa5..bc5b7584140 100644 --- a/ortools/linear_solver/java/linear_solver.i +++ b/ortools/linear_solver/java/linear_solver.i @@ -46,6 +46,9 @@ class MPModelRequest; class MPSolutionResponse; } // namespace operations_research +// cross-language polymorphism should be enabled to support MPCallback feature +%module(directors="1") operations_research; + %{ #include "ortools/linear_solver/linear_solver.h" #include "ortools/linear_solver/model_exporter.h" @@ -512,6 +515,39 @@ PROTO2_RETURN( %rename (ShowUnusedVariables) operations_research::MPModelExportOptions::show_unused_variables; %rename (MaxLineLength) operations_research::MPModelExportOptions::max_line_length; +// Expose the MPCallback & MPCallbackContext APIs +// Enable cross-language polymorphism for MPCallback virtual class +%feature("director") operations_research::MPCallback; +%unignore operations_research::MPCallback; +%unignore operations_research::MPCallback::MPCallback; +%unignore operations_research::MPCallback::~MPCallback; +%rename (runCallback) operations_research::MPCallback::RunCallback; +%rename (mightAddCuts) operations_research::MPCallback::might_add_cuts; +%rename (mightAddLazyConstraints) operations_research::MPCallback::might_add_lazy_constraints; +%unignore operations_research::MPCallbackContext; +%unignore operations_research::MPCallbackContext::MPCallbackContext; +%unignore operations_research::MPCallbackContext::~MPCallbackContext; +%unignore operations_research::MPCallbackEvent; +%rename (UNKNOWN) operations_research::MPCallbackEvent::kUnknown; +%rename (POLLING) operations_research::MPCallbackEvent::kPolling; +%rename (PRESOLVE) operations_research::MPCallbackEvent::kPresolve; +%rename (SIMPLEX) operations_research::MPCallbackEvent::kSimplex; +%rename (MIP) operations_research::MPCallbackEvent::kMip; +%rename (MIP_SOLUTION) operations_research::MPCallbackEvent::kMipSolution; +%rename (MIP_NODE) operations_research::MPCallbackEvent::kMipNode; +%rename (BARRIER) operations_research::MPCallbackEvent::kBarrier; +%rename (MESSAGE) operations_research::MPCallbackEvent::kMessage; +%rename (MULTI_OBJ) operations_research::MPCallbackContext::MPCallbackEvent::kMultiObj; +%rename (event) operations_research::MPCallbackContext::Event; +%rename (canQueryVariableValues) operations_research::MPCallbackContext::CanQueryVariableValues; +%rename (variableValue) operations_research::MPCallbackContext::VariableValue; +%rename (addCut) operations_research::MPCallbackContext::AddCut; +%rename (addLazyConstraint) operations_research::MPCallbackContext::AddLazyConstraint; +%rename (suggestSolution) operations_research::MPCallbackContext::SuggestSolution; +%rename (numExploredNodes) operations_research::MPCallbackContext::NumExploredNodes; +%rename (setCallback) operations_research::MPSolver::SetCallback; + +%include "ortools/linear_solver/linear_solver_callback.h" %include "ortools/linear_solver/linear_solver.h" %include "ortools/linear_solver/model_exporter.h" diff --git a/ortools/linear_solver/linear_solver_callback.h b/ortools/linear_solver/linear_solver_callback.h index 6c6a0713be3..b0f6607670a 100644 --- a/ortools/linear_solver/linear_solver_callback.h +++ b/ortools/linear_solver/linear_solver_callback.h @@ -137,7 +137,7 @@ class MPCallbackContext { // class MPCallback { public: - // If you intend to call call MPCallbackContext::AddCut(), you must set + // If you intend to call MPCallbackContext::AddCut(), you must set // might_add_cuts below to be true. Likewise for // MPCallbackContext::AddLazyConstraint() and might_add_lazy_constraints. MPCallback(bool might_add_cuts, bool might_add_lazy_constraints) @@ -153,6 +153,7 @@ class MPCallback { bool might_add_cuts() const { return might_add_cuts_; } bool might_add_lazy_constraints() const { return might_add_lazy_constraints_; + } private: diff --git a/ortools/linear_solver/python/linear_solver.i b/ortools/linear_solver/python/linear_solver.i index 98e37728d6f..26f4ad335d6 100644 --- a/ortools/linear_solver/python/linear_solver.i +++ b/ortools/linear_solver/python/linear_solver.i @@ -47,11 +47,25 @@ class MPSolutionResponse; class IISResponse; } // namespace operations_research +// cross-language polymorphism should be enabled to support MPCallback feature +%module(directors="1") operations_research; +%feature("director:except") { +if ($error != NULL) { + throw Swig::DirectorMethodException(); +} +} + +%exception { +try { $action } +catch (Swig::DirectorException &e) { SWIG_fail; } +} + %{ #include "ortools/linear_solver/linear_solver.h" #include "ortools/linear_solver/model_exporter.h" #include "ortools/linear_solver/model_exporter_swig_helper.h" #include "ortools/linear_solver/model_validator.h" +#include "ortools/linear_solver/python/python_mp_callback.h" %} %pythoncode %{ @@ -275,6 +289,12 @@ PY_CONVERT(MPConstraint); PY_CONVERT_HELPER_PTR(MPVariable); PY_CONVERT(MPVariable); +PY_CONVERT_HELPER_PTR(PythonMPCallback); +PY_CONVERT(PythonMPCallback); + +PY_CONVERT_HELPER_PTR(MPCallbackContext); +PY_CONVERT(MPCallbackContext); + %ignoreall %unignore operations_research; @@ -494,7 +514,41 @@ PY_CONVERT(MPVariable); // Expose the model validator. %rename (FindErrorInModelProto) operations_research::FindErrorInMPModelProto; +// Expose the MPCallback & MPCallbackContext APIs +// Enable cross-language polymorphism for MPCallback virtual class +%feature("director") operations_research::PythonMPCallback; +%rename (MPCallback) operations_research::PythonMPCallback; +%unignore operations_research::PythonMPCallback::PythonMPCallback; +%unignore operations_research::PythonMPCallback::~PythonMPCallback; +%unignore operations_research::PythonMPCallback::Run; +%unignore operations_research::PythonMPCallback::might_add_cuts; +%unignore operations_research::PythonMPCallback::might_add_lazy_constraints; +%unignore operations_research::MPCallbackContext; +%unignore operations_research::MPCallbackContext::MPCallbackContext; +%unignore operations_research::MPCallbackContext::~MPCallbackContext; +%unignore operations_research::MPCallbackEvent; +%rename (UNKNOWN) operations_research::MPCallbackEvent::kUnknown; +%rename (POLLING) operations_research::MPCallbackEvent::kPolling; +%rename (PRESOLVE) operations_research::MPCallbackEvent::kPresolve; +%rename (SIMPLEX) operations_research::MPCallbackEvent::kSimplex; +%rename (MIP) operations_research::MPCallbackEvent::kMip; +%rename (MIP_SOLUTION) operations_research::MPCallbackEvent::kMipSolution; +%rename (MIP_NODE) operations_research::MPCallbackEvent::kMipNode; +%rename (BARRIER) operations_research::MPCallbackEvent::kBarrier; +%rename (MESSAGE) operations_research::MPCallbackEvent::kMessage; +%rename (MULTI_OBJ) operations_research::MPCallbackContext::MPCallbackEvent::kMultiObj; +%unignore operations_research::MPCallbackContext::Event; +%unignore operations_research::MPCallbackContext::CanQueryVariableValues; +%unignore operations_research::MPCallbackContext::VariableValue; +%unignore operations_research::MPCallbackContext::AddCut; +%unignore operations_research::MPCallbackContext::AddLazyConstraint; +%unignore operations_research::MPCallbackContext::SuggestSolution; +%unignore operations_research::MPCallbackContext::NumExploredNodes; +%unignore operations_research::MPSolver::SetCallback; + +%include "ortools/linear_solver/linear_solver_callback.h" %include "ortools/linear_solver/linear_solver.h" +%include "ortools/linear_solver/python/python_mp_callback.h" %include "ortools/linear_solver/model_exporter.h" %include "ortools/linear_solver/model_exporter_swig_helper.h" diff --git a/ortools/linear_solver/python/python_mp_callback.h b/ortools/linear_solver/python/python_mp_callback.h new file mode 100644 index 00000000000..24399140172 --- /dev/null +++ b/ortools/linear_solver/python/python_mp_callback.h @@ -0,0 +1,30 @@ +// +// Created by mitripet on 24/11/23. +// + +#include "ortools/linear_solver/linear_solver.h" + +#ifndef ORTOOLS_PYTHON_MP_CALLBACK_H +#define ORTOOLS_PYTHON_MP_CALLBACK_H + +namespace operations_research { +// TODO : add description +class PythonMPCallback : public MPCallback { + public: + PythonMPCallback(bool might_add_cuts, bool might_add_lazy_constraints) + : MPCallback(might_add_cuts, might_add_lazy_constraints){}; + virtual ~PythonMPCallback() {} + + virtual void Run(MPCallbackContext* callback_context) = 0; + + void RunCallback(MPCallbackContext* callback_context) override { + // acquire Python GIL before running user-defined callback in python + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + Run(callback_context); + // release Python GIL + PyGILState_Release(gstate); + }; +}; +} +#endif // ORTOOLS_PYTHON_MP_CALLBACK_H diff --git a/examples/xpress_tests/callback_xpress.py b/ortools/linear_solver/python/xpress_callback_test.py similarity index 91% rename from examples/xpress_tests/callback_xpress.py rename to ortools/linear_solver/python/xpress_callback_test.py index 7ac4bc71f8a..ae4dad4049f 100644 --- a/examples/xpress_tests/callback_xpress.py +++ b/ortools/linear_solver/python/xpress_callback_test.py @@ -14,7 +14,7 @@ """MIP example/test that shows how to use the callback API.""" from ortools.linear_solver import pywraplp -from ortools.linear_solver.pywraplp import MPCallbackContext, MPCallback +from ortools.linear_solver.pywraplp import MPCallbackContext, MPCallback, MPCallbackEvent_MIP_SOLUTION import random import unittest @@ -26,14 +26,16 @@ def __init__(self, mp_solver: pywraplp.Solver): self._solutions_ = 0 self._last_var_values_ = [0] * len(mp_solver.variables()) - def RunCallback(self, ctx: MPCallbackContext): + def Run(self, ctx: MPCallbackContext): + if ctx.Event() != MPCallbackEvent_MIP_SOLUTION: + return self._solutions_ += 1 for i in range(0, len(self._mp_solver_.variables())) : self._last_var_values_[i] = ctx.VariableValue(self._mp_solver_.variable(i)) -class TestSiriusXpress(unittest.TestCase): - def test_callback(self): +class callback(unittest.TestCase): + def test_xpress_newmipsol_cb(self): """Builds a large MIP that is difficult to solve, in order for us to have time to intercept non-optimal feasible solutions using callback""" solver = pywraplp.Solver.CreateSolver('XPRESS_MIXED_INTEGER_PROGRAMMING') @@ -71,10 +73,9 @@ def test_callback(self): # solutions (tested with XPRESS v9.0, may change in later versions) self.assertTrue(cb._solutions_ > 5) # Test that the last solution intercepted by callback is the same as the optimal one retained - for i in range(0, len(solver.variables())): + for i in range(0, solver.NumVariables()): self.assertAlmostEqual(cb._last_var_values_[i], solver.variable(i).SolutionValue()) if __name__ == '__main__': unittest.main() - diff --git a/ortools/linear_solver/xpress_interface.cc b/ortools/linear_solver/xpress_interface.cc index c28cab27fdf..c473b4eccb6 100644 --- a/ortools/linear_solver/xpress_interface.cc +++ b/ortools/linear_solver/xpress_interface.cc @@ -263,8 +263,8 @@ class MPCallbackWrapper { explicit MPCallbackWrapper(MPCallback* callback) : callback_(callback){}; MPCallback* GetCallback() const { return callback_; } // Since our (C++) call-back functions are called from the XPRESS (C) code, - // exceptions thrown in our call-back code are not caught by XPRESS. - // We have to catch them, interrupt XPRESS, and re-throw them after XPRESS is + // exceptions thrown in our call-back code are not caught by XPRESS. + // We have to catch them, interrupt XPRESS, and log them after XPRESS is // effectively interrupted (ie after solve). void CatchException(XPRSprob cbprob) { exceptions_mutex_.lock(); @@ -272,13 +272,17 @@ class MPCallbackWrapper { interruptXPRESS(cbprob, CALLBACK_EXCEPTION); exceptions_mutex_.unlock(); } - void RethrowCaughtExceptions() { + void LogCaughtExceptions() { exceptions_mutex_.lock(); for (const std::exception_ptr& ex : caught_exceptions_) { try { std::rethrow_exception(ex); - } catch (std::bad_exception const&) { - LOG(ERROR) << "Bad exception"; + } catch (std::exception &ex) { + // We don't want the interface to throw exceptions, plus it causes + // SWIG issues in Java & Python. Instead, we'll only log them. + // (The use cases where the user has to raise an exception inside their + // call-back does not seem to be frequent, anyway.) + LOG(ERROR) << "Caught exception during user-defined call-back: " << ex.what(); } } caught_exceptions_.clear(); @@ -1926,7 +1930,7 @@ MPSolver::ResultStatus XpressInterface::Solve(MPSolverParameters const& param) { } if (mp_callback_wrapper != nullptr) { - mp_callback_wrapper->RethrowCaughtExceptions(); + mp_callback_wrapper->LogCaughtExceptions(); delete mp_callback_wrapper; } diff --git a/ortools/linear_solver/xpress_interface_test.cc b/ortools/linear_solver/xpress_interface_test.cc index 4df44841be8..fe1b6dac8b0 100644 --- a/ortools/linear_solver/xpress_interface_test.cc +++ b/ortools/linear_solver/xpress_interface_test.cc @@ -1286,22 +1286,17 @@ TEST(XpressInterface, SetAndResetCallBack) { } TEST(XpressInterface, CallbackThrowsException) { - // Test that when the callback throws an exception, it is caught and re-thrown + // Test that when the callback throws an exception, it is caught and logged UNITTEST_INIT_MIP(); auto oldMpCallback = buildLargeMipWithCallback(solver, 30, 30); auto newMpCallback = new MyMPCallback(&solver, true); solver.SetCallback((MPCallback*)newMpCallback); - EXPECT_THROW( - { - try { - solver.Solve(); - } catch (const std::runtime_error& e) { - // this tests that it has the correct message - EXPECT_STREQ("This is a mocked exception in MyMPCallback", e.what()); - throw; - } - }, - std::runtime_error); + testing::internal::CaptureStderr(); + EXPECT_NO_THROW(solver.Solve()); + std::string errors = testing::internal::GetCapturedStderr(); + // Test that StdErr contains the following error message + std::string expected_error = "Caught exception during user-defined call-back: This is a mocked exception in MyMPCallback"; + ASSERT_NE(errors.find(expected_error), std::string::npos); } } // namespace operations_research