From e0205b552b077f7460e814cf621fba6a86209901 Mon Sep 17 00:00:00 2001 From: daymontas1 <53146798+daymontas1@users.noreply.github.com> Date: Fri, 22 Mar 2024 20:52:44 +0100 Subject: [PATCH 1/6] Update macro_solve.gms An if statement has been introduced to allow users to select whether they want to solve sequentially or concurrently for all regions. This is determined by the ''solve_param'' parameter, defined in ''MESSAGE-MACRO_run''. The default solving is sequential (for ''solve_param''=1), so if the user doesn't update the solving mechanism, the macro is solved as it used to be. In terms of efficiency, the concurrent solving is slightly more efficient. For the concurrent solving mechanism (for ''solve_param''=2), all nodes are activated from the beginning, thereby enabling simultaneous solving for all regions. This setting allows for factoring in interregional interactions into the optimization process. We want this concurrent optimization option, as we intend to incorporate interregional feedback mechanisms into MACRO, e.g., interregional investments. Given that in the current version of MACRO regions are independent, both solving mechanisms should produce the same results. --- message_ix/model/MACRO/macro_solve.gms | 38 ++++++++++++++------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/message_ix/model/MACRO/macro_solve.gms b/message_ix/model/MACRO/macro_solve.gms index 86933c9a3..c88324239 100755 --- a/message_ix/model/MACRO/macro_solve.gms +++ b/message_ix/model/MACRO/macro_solve.gms @@ -38,30 +38,32 @@ C.FX(node_macro, macro_base_period) = c0(node_macro) ; I.FX(node_macro, macro_base_period) = i0(node_macro) ; EC.FX(node_macro, macro_base_period) = y0(node_macro) - i0(node_macro) - c0(node_macro) ; -* ------------------------------------------------------------------------------ +* Conditional execution based on solve_param +if (solve_param = 1, * solving the model region by region -* ------------------------------------------------------------------------------ - -node_active(node) = no ; - -LOOP(node $ node_macro(node), - - node_active(node_macro) = no ; - node_active(node) = YES; -* DISPLAY node_active ; - -* ------------------------------------------------------------------------------ + node_active(node) = no ; + LOOP(node $ node_macro(node), + node_active(node_macro) = no ; + node_active(node) = YES; +* DISPLAY node_active ; * solve statement -* ------------------------------------------------------------------------------ - - SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP ; - + SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP ; * write model status summary (by node) * status(node,'modelstat') = MESSAGE_MACRO.modelstat ; * status(node,'solvestat') = MESSAGE_MACRO.solvestat ; * status(node,'resUsd') = MESSAGE_MACRO.resUsd ; * status(node,'objEst') = MESSAGE_MACRO.objEst ; * status(node,'objVal') = MESSAGE_MACRO.objVal ; - + ) +elseif (solve_param = 2), +*Solving model with all regions concurrently + node_active(node_macro) = YES; +* solve statement + SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP; +* write model status summary +* status('all','modelstat') = MESSAGE_MACRO.modelstat; +* status('all','solvestat') = MESSAGE_MACRO.solvestat; +* status('all','resUsd') = MESSAGE_MACRO.resUsd; +* status('all','objEst') = MESSAGE_MACRO.objEst; +* status('all','objVal') = MESSAGE_MACRO.objVal; ) ; - From 98b4b43d7d7ab07a1993875f95fd014726c6135b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 20 Jan 2025 23:25:20 +0100 Subject: [PATCH 2/6] Parametrize one MACRO test for concurrent {0, 1} --- message_ix/tests/test_macro.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/message_ix/tests/test_macro.py b/message_ix/tests/test_macro.py index e4d9ec6d9..1d906c46f 100644 --- a/message_ix/tests/test_macro.py +++ b/message_ix/tests/test_macro.py @@ -281,9 +281,19 @@ def test_calibrate(westeros_solved, w_data_path): assert not end_grow.isnull().any() -def test_calibrate_roundtrip(westeros_solved, w_data_path): +@pytest.mark.parametrize( + "kwargs", + ( + {}, # Default concurrent=0 + dict(concurrent=0), # Explicit value, same as default + dict(concurrent=1), + ), +) +def test_calibrate_roundtrip(westeros_solved, w_data_path, kwargs): # this is a regression test with values observed on May 23, 2024 - with_macro = westeros_solved.add_macro(w_data_path, check_convergence=True) + with_macro = westeros_solved.add_macro( + w_data_path, check_convergence=True, **kwargs + ) aeei = with_macro.par("aeei")["value"].values npt.assert_allclose( aeei, From e5677b378e7c02ca0750a05eb1b68cb6f81ba2cb Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 20 Jan 2025 23:27:15 +0100 Subject: [PATCH 3/6] Add MACRO/setup.gms Parallel to MESSAGE/model_setup.gms, contains variables for conditional compilation. --- message_ix/model/MACRO/setup.gms | 21 +++++++++++++++++++++ message_ix/model/MACRO_run.gms | 11 +++-------- message_ix/model/MESSAGE-MACRO_run.gms | 4 ++++ message_ix/model/MESSAGE_master.gms | 10 ++++++---- 4 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 message_ix/model/MACRO/setup.gms diff --git a/message_ix/model/MACRO/setup.gms b/message_ix/model/MACRO/setup.gms new file mode 100644 index 000000000..508479101 --- /dev/null +++ b/message_ix/model/MACRO/setup.gms @@ -0,0 +1,21 @@ +* A scenario name is mandatory to load the gdx file - abort the run if not specified or file does not exist +$IF NOT SET in $ABORT "no input data file provided!" +$IF NOT EXIST '%in%' $ABORT "input GDX file '%in%' does not exist!" +$IF NOT SET out $SETGLOBAL out "output/MsgOutput.gdx" + +* MACRO mode. This can take 3 possible values: +* +* - "none": MACRO is not run, MESSAGE is run in stand-alone mode. +* - "linked": MESSAGE and MACRO are run in linked/iterative mode. +* This value is set in MESSAGE-MACRO_run.gms. +* - "standalone": MACRO is run without MESSAGE. +* This value is set in MACRO_run.gms +$IF NOT SET macromode $ABORT "The global setting/command line option --macromode must be set" + +* Option to solve MACRO either… +* +* - 0: in sequence +* - 1: in parallel +* +* See macro_solve.gms +$IF NOT SET MACRO_CONCURRENT $SETGLOBAL MACRO_CONCURRENT "0" diff --git a/message_ix/model/MACRO_run.gms b/message_ix/model/MACRO_run.gms index 1ad4dc2d2..4ff91553b 100644 --- a/message_ix/model/MACRO_run.gms +++ b/message_ix/model/MACRO_run.gms @@ -1,17 +1,12 @@ -* a scenario name is mandatory to load the gdx file - abort the run if not specified or file does not exist -$IF NOT SET in $ABORT "no input data file provided!" -$IF NOT EXIST '%in%' $ABORT "input GDX file '%in%' does not exist!" - -** option to run MACRO standalone or interactively linked (iterating) with MESSAGE ** -*$SETGLOBAL macromode "linked" +* Run MACRO in stand-alone mode, without MESSAGE. +* To run coupled with MESSAGE, use MESSAGE-MACRO_run.gms instead of this file. $SETGLOBAL macromode "standalone" +$INCLUDE MACRO/setup.gms $INCLUDE MACRO/macro_data_load.gms $INCLUDE MACRO/macro_core.gms $INCLUDE MACRO/macro_calibration.gms $INCLUDE MACRO/macro_reporting.gms * dump all input data, processed data and results to a gdx file (with additional comment as name extension if provided) -$IF NOT SET out $SETGLOBAL out "output/MsgOutput.gdx" execute_unload "%out%" - diff --git a/message_ix/model/MESSAGE-MACRO_run.gms b/message_ix/model/MESSAGE-MACRO_run.gms index 36cb49217..ae39e7c1a 100644 --- a/message_ix/model/MESSAGE-MACRO_run.gms +++ b/message_ix/model/MESSAGE-MACRO_run.gms @@ -26,6 +26,9 @@ $INCLUDE includes/copyright.gms * (or ``model\output\MsgOutput.gdx`` if ``--out`` is not provided). *** +* Run MESSAGE and MACRO in linked/iterative mode mode. +* To run MACRO alone, use MACRO_run.gms instead of this file. +* To run MESSAGE alone, use MESSAGE_run.gms or MESSAGE_master.gms instead of this file. $SETGLOBAL macromode "linked" $EOLCOM # $INCLUDE MESSAGE/model_setup.gms @@ -34,6 +37,7 @@ $INCLUDE MESSAGE/model_setup.gms * load additional equations and parameters for MACRO * *----------------------------------------------------------------------------------------------------------------------* +$INCLUDE MACRO/setup.gms $INCLUDE MACRO/macro_data_load.gms $INCLUDE MACRO/macro_core.gms diff --git a/message_ix/model/MESSAGE_master.gms b/message_ix/model/MESSAGE_master.gms index 67175c84f..eeafeb8d2 100644 --- a/message_ix/model/MESSAGE_master.gms +++ b/message_ix/model/MESSAGE_master.gms @@ -12,10 +12,12 @@ $ONGLOBAL ** scenario/case selection - this must match the name of the MsgData_<%%%>.gdx input data file ** $SETGLOBAL data "" -** MACRO mode -* "none": MESSAGEix is run in stand-alone mode -* "linked": MESSAGEix-MACRO is run in iterative mode ** -$SETGLOBAL macromode "none" +* MACRO mode. This can take 3 possible values, only 2 of which are usable with this file: +* +* - "none": MACRO is not run, MESSAGE is run in stand-alone mode. +* - "linked": MESSAGE and MACRO are run in linked/iterative mode. +* - "standalone": MACRO is run without MESSAGE. Not valid when using this file; use MACRO_run.gms instead. +$IF NOT SET macromode $SETGLOBAL macromode "none" ** define the time horizon over which the model optimizes (perfect foresight, myopic or rolling horizon) ** * perfect foresight - 0 From bdd136684b5cdb4a8f34c5713a8051b0d922cbf9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 20 Jan 2025 23:27:47 +0100 Subject: [PATCH 4/6] Make MACRO_CONCURRENT a GAMS compile-time variable --- message_ix/model/MACRO/macro_solve.gms | 56 +++++++++++++++----------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/message_ix/model/MACRO/macro_solve.gms b/message_ix/model/MACRO/macro_solve.gms index c88324239..1d20626ef 100755 --- a/message_ix/model/MACRO/macro_solve.gms +++ b/message_ix/model/MACRO/macro_solve.gms @@ -38,32 +38,40 @@ C.FX(node_macro, macro_base_period) = c0(node_macro) ; I.FX(node_macro, macro_base_period) = i0(node_macro) ; EC.FX(node_macro, macro_base_period) = y0(node_macro) - i0(node_macro) - c0(node_macro) ; -* Conditional execution based on solve_param -if (solve_param = 1, -* solving the model region by region - node_active(node) = no ; - LOOP(node $ node_macro(node), - node_active(node_macro) = no ; - node_active(node) = YES; -* DISPLAY node_active ; -* solve statement - SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP ; -* write model status summary (by node) -* status(node,'modelstat') = MESSAGE_MACRO.modelstat ; -* status(node,'solvestat') = MESSAGE_MACRO.solvestat ; -* status(node,'resUsd') = MESSAGE_MACRO.resUsd ; -* status(node,'objEst') = MESSAGE_MACRO.objEst ; -* status(node,'objVal') = MESSAGE_MACRO.objVal ; - ) -elseif (solve_param = 2), -*Solving model with all regions concurrently - node_active(node_macro) = YES; -* solve statement - SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP; -* write model status summary +$IFTHEN %MACRO_CONCURRENT% == "0" + +DISPLAY "Solve MACRO for each node in sequence"; + +node_active(node) = NO ; + +LOOP(node$node_macro(node), + node_active(node_macro) = NO ; + node_active(node) = YES ; +* DISPLAY node_active ; + + SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP ; + +* Write model status summary for the current node +* status(node,'modelstat') = MESSAGE_MACRO.modelstat ; +* status(node,'solvestat') = MESSAGE_MACRO.solvestat ; +* status(node,'resUsd') = MESSAGE_MACRO.resUsd ; +* status(node,'objEst') = MESSAGE_MACRO.objEst ; +* status(node,'objVal') = MESSAGE_MACRO.objVal ; +); + +$ELSE + +DISPLAY "Solve MACRO for all nodes concurrently"; + +node_active(node_macro) = YES; + +SOLVE MESSAGE_MACRO MAXIMIZING UTILITY USING NLP; + +* Write model status summary for all nodes * status('all','modelstat') = MESSAGE_MACRO.modelstat; * status('all','solvestat') = MESSAGE_MACRO.solvestat; * status('all','resUsd') = MESSAGE_MACRO.resUsd; * status('all','objEst') = MESSAGE_MACRO.objEst; * status('all','objVal') = MESSAGE_MACRO.objVal; -) ; + +$ENDIF From ab2ba14efe76da495d1aecb6b7edf583dbeaf220 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 20 Jan 2025 23:29:19 +0100 Subject: [PATCH 5/6] =?UTF-8?q?Handle=20concurrent=3D=E2=80=A6=20arg=20to?= =?UTF-8?q?=20MACRO=20solve()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message_ix/models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/message_ix/models.py b/message_ix/models.py index 057f50e94..9ea31b441 100644 --- a/message_ix/models.py +++ b/message_ix/models.py @@ -912,8 +912,21 @@ def __init__(self, *args, **kwargs): f"{self.name} requires GAMS >= {self.GAMS_min_version}; found {version}" ) + # Additional command-line arguments to GAMS + solve_args = [] + try: + concurrent = str(kwargs.pop("concurrent")) + except KeyError: + pass + else: + if concurrent not in ("0", "1"): + raise ValueError(f"{concurrent = }") + solve_args.append(f"--MACRO_CONCURRENT={concurrent}") + super().__init__(*args, **kwargs) + self.solve_args.extend(solve_args) + @classmethod def initialize(cls, scenario, with_data=False): """Initialize the model structure.""" From 2da69693e6edf9d305f47ee4f4a72d768358167a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 20 Jan 2025 23:36:08 +0100 Subject: [PATCH 6/6] Add #808 to docs, release notes --- RELEASE_NOTES.rst | 1 + doc/api.rst | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 380a7173a..80b11a54b 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -30,6 +30,7 @@ All changes - Support for Python 3.8 is dropped (:pull:`881`), as it has reached end-of-life. - Add :meth:`.Reporter.add_sankey` and :mod:`.tools.sankey` to create Sankey diagrams from solved scenarios (:pull:`770`). The :file:`westeros_sankey.ipynb` :ref:`tutorial ` shows how to use this feature. +- Add the :py:`concurrent=...` model option to :class:`.MACRO` (:pull:`808`). - Add option to :func:`.util.copy_model` from a non-default location of model files (:pull:`877`). - Bug fix for calculation of ``PRICE_EMISSION`` (:pull:`912`, :issue:`723`). See the :ref:`migration note ` above. diff --git a/doc/api.rst b/doc/api.rst index e17e468a6..659eb8325 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -260,6 +260,14 @@ Model classes :exclude-members: items :show-inheritance: + The MACRO class solves only the MACRO model in “standalone” mode—that is, without MESSAGE. + It is also invoked from :class:`.MESSAGE_MACRO` to process *model_options* to control the behaviour of MACRO: + + - **concurrent** (:class:`int` or :class:`float`, either :py:`0` or :py:`1`). + This corresponds to the GAMS compile-time variable ``MACRO_CONCURRENT``. + If set to :py:`0` (the default), MACRO is solved in a loop, once for each node in the Scenario. + If set to :py:`1`, MACRO is solved only once, for all nodes simultaneously. + .. autoattribute:: items :no-value: