Pull request #321: GCAM 7.0 Bugfix

Merge in JGCRI/gcam-core from enl/bugfix/gcam7-bugfix to master

* commit '29a5597d2d8660af82be5045ef73d4a6cc1e9f79': (48 commits)
  Update Macro productivity params for gcam-v7.1
  Include one solution tweak helps when the Broyden solver is bouncing around a solution and having trouble.  It may inflate the number of iterations slightly in the general case, however.
  Remove the old commented out electricity.xml from the configuration file as it is now deprecated
  Bump version number
  Update target finder initial guesses
  Fix failure in usermod chunks due to behavior change in recent versions of R
  Fix link in README and add link to community guidelines
  Remove FUSION_VECTOR_SIZE in VS config as well
  Get rid of FUSION_MAX_VECTOR_SIZE which isn't needed anymore
  Address PR comments
  Addressing PR comments
  Update to the latest package versions which seem to be still shifting post R 4.4
  Some more query fixes for primary energy and internal gains
  Fix bug AgStorage which can introduce a "stale" value calculation which produced slightly different round off error on subsequent World.calc
  Fixes to get tests passing and reduce warnings
  Update module-helpers.R to 1) remove a function that is not used and 2) update threshold of outlier replacement function
  Point to testing-framework version that uses R 4.4
  Ensure the propmised XML name and the name set in create_xml match for ssp2_emissions_factor_TradBio.xml as drake has to rely on the former. We will eventually enforce this with checks.
  Now that we have harmonized the 1990 -> 1975 GDP deflator between the C++ and gcamdata we no longer require the hard coded GCAM value in gcamdata
  Put in a scaler for biomass nonLandVariableCost in the final calibration period as a way to deal with mechanical issue of negative profit rates for calibrating ghost shares weights.
  ...

README.md

@@ -52,8 +52,9 @@ mitigation policy.
 
 * [GCAM Documentation](http://jgcri.github.io/gcam-doc/)
 * [Getting Started with GCAM](http://jgcri.github.io/gcam-doc/user-guide.html)
-* [GCAM Community](http://www.globalchange.umd.edu/models/gcam/gcam-community/)
+* [GCAM Community](https://gcims.pnnl.gov/community)
 * [GCAM Videos and Tutorial Slides](https://gcims.pnnl.gov/community)
+* [GCAM Citation and Co-authorship Guidelines](http://jgcri.github.io/gcam-doc/community-guide.html)
 
 ## Selected Publications
 

cvs/objects/build/linux/config.system

@@ -17,6 +17,9 @@ TBB_INCLUDE     = $(GCAMLIB_HOME)/include/oneapi
 TBB_LIB	        = $(GCAMLIB_HOME)/lib
 JAVA_INCLUDE    = ${JAVA_HOME}/include
 JAVA_LIB        = ${JAVA_HOME}/jre/lib/amd64/server
+ifeq (openjdk,$(findstring openjdk,$(JAVA_HOME)))
+JAVA_LIB        = ${JAVA_HOME}/lib/server
+endif
 JARS_LIB        = $(GCAMLIB_HOME)/lib/jars/*
 ## `module load mkl/15.0.1` will get you the following variables:
 ifdef MLIB_CFLAGS

cvs/objects/build/linux/configure.gcam

@@ -12,7 +12,7 @@ ifeq ($(strip $(CC)),)
 CC		= gcc
 endif
 CXXOPTIM        = -O3 -pthread
-CXXDEBUG        = -ggdb -DNDEBUG -DFUSION_MAX_VECTOR_SIZE=30
+CXXDEBUG        = -ggdb -DNDEBUG
 CXXBASEOPTS     = $(CXXDEBUG)
 
 ARCH_FLAGS      = 
@@ -123,7 +123,9 @@ ifeq ($(strip $(JARS_LIB)),)
   $(error Unable to detect Jar lib path, please set env variable JARS_LIB)
 endif
 ## custom values set by environment variables
-OSNAME_LOWERCASE := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+ifndef OSNAME_LOWERCASE
+  OSNAME_LOWERCASE := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+endif
 JAVAINC = -I$(JAVA_INCLUDE) -I$(JAVA_INCLUDE)/$(OSNAME_LOWERCASE)
 JAVALIB = -L$(JAVA_LIB)
 JAVA_RPATH = -Wl,-rpath,$(JAVA_LIB)
@@ -158,7 +160,7 @@ LIBDIR		= -L/usr/local/lib -L$(BUILDPATH) $(JAVALIB) $(TBB_LIBRARY)
 ### The rest should be mostly compiler independent
 ## Note $(PROF) will be set as needed if we are building the gcam-prof target
 CPPFLAGS	= $(INCLUDE) $(ARCH_FLAGS) $(JARSLIB) -DGCAM_PARALLEL_ENABLED=$(USE_GCAM_PARALLEL) -DUSE_HECTOR=$(USE_HECTOR) $(MKL_CFLAGS)
-CXXFLAGS        = $(CXXOPTIM) $(CXXBASEOPTS) $(PROF) -MMD -std=$(CXXSTD) -Wno-deprecated
+CXXFLAGS        = $(CXXOPTIM) $(CXXBASEOPTS) $(PROF) $(CXXEXTRA) -MMD -std=$(CXXSTD) -Wno-deprecated
 FCFLAGS         = $(FCOPTIM) $(FCBASEOPTS) $(PROF)
 LD              = $(CXX) $(PROF)
 LDFLAGS         = $(CXXFLAGS) $(JAVA_RPATH) $(TBB_RPATH) $(MKL_LDFLAGS)

cvs/objects/build/vc10/objects.vcxproj

@@ -372,7 +372,7 @@
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
       <OmitFramePointers>true</OmitFramePointers>
       <AdditionalIncludeDirectories>..\..\..\..\libs\boost-lib;..\..\..\..\libs\java\include;..\..\..\..\libs\java\include\win32;..\..;..\..\climate\source\hector\headers;..\..\..\..\libs\tbb\include;..\..\..\..\libs\Eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;BOOST_DATE_TIME_NO_LIB;BOOST_IOSTREAMS_NO_LIB;FUSION_MAX_VECTOR_SIZE=30;BOOST_MATH_TR1_NO_LIB;NDEBUG;EIGEN_MPL2_ONLY;GCAM_PARALLEL_ENABLED;_WINDOWS;_AFXDLL;NOGDI;NOMINMAX;JARS_LIB#"../libs/basex/BaseX.jar\x3B../libs/jars/*";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;BOOST_DATE_TIME_NO_LIB;BOOST_IOSTREAMS_NO_LIB;BOOST_MATH_TR1_NO_LIB;NDEBUG;EIGEN_MPL2_ONLY;GCAM_PARALLEL_ENABLED;_WINDOWS;_AFXDLL;NOGDI;NOMINMAX;JARS_LIB#"../libs/basex/BaseX.jar\x3B../libs/jars/*";%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <FunctionLevelLinking>true</FunctionLevelLinking>

cvs/objects/build/xcode3/objects.xcodeproj/project.pbxproj

@@ -2519,7 +2519,6 @@
 				GCC_OPTIMIZATION_LEVEL = 0;
 				GCC_PREPROCESSOR_DEFINITIONS = (
 					"JARS_LIB=\"\\\"../libs/basex/BaseX.jar:../libs/jars/*\\\"\"",
-					"FUSION_MAX_VECTOR_SIZE=30",
 				);
 				GCC_UNROLL_LOOPS = YES;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES;
@@ -2562,7 +2561,6 @@
 				GCC_PREPROCESSOR_DEFINITIONS = (
 					NDEBUG,
 					"JARS_LIB=\"\\\"../libs/basex/BaseX.jar:../libs/jars/*\\\"\"",
-					"FUSION_MAX_VECTOR_SIZE=30",
 					EIGEN_MPL2_ONLY,
 					BOOST_IOSTREAMS_NO_LIB,
 				);

cvs/objects/containers/include/national_account.h

@@ -112,6 +112,7 @@ class NationalAccount: public IVisitable, public IYeared, public AParsable
         INVESTMENT,
         DEPRECIATION,
         CAPITAL_STOCK,
+        CAPITAL_VALUE,
         CAPITAL_ENERGY_INV,
         CONSUMER_DURABLE_INV,
         GDP,

cvs/objects/containers/source/national_account.cpp

@@ -109,6 +109,9 @@ bool NationalAccount::XMLParse( rapidxml::xml_node<char>* & aNode ) {
     else if  ( nodeName == enumToXMLName( CAPITAL_STOCK ) ) {
         setAccount( CAPITAL_STOCK, XMLParseHelper::getValue<double>( aNode ) );
     }
+    else if  ( nodeName == enumToXMLName( CAPITAL_VALUE ) ) {
+        setAccount( CAPITAL_VALUE, XMLParseHelper::getValue<double>( aNode ) );
+    }
     else if  ( nodeName == enumToXMLName( DEPRECIATION_RATE ) ) {
         setAccount( DEPRECIATION_RATE, XMLParseHelper::getValue<double>( aNode ) );
     }
@@ -287,6 +290,7 @@ const string& NationalAccount::enumToXMLName( const AccountType aType ) const {
             "investment",
             "depreciation",
             "capital-stock",
+            "capital-value",
             "energy-investment",
             "consumer-durable",
             "GDP",

cvs/objects/containers/source/single_scenario_runner.cpp

@@ -147,7 +147,7 @@ bool SingleScenarioRunner::setupScenarios( Timer& timer,
 
     // Add to all loggers that a new scenario is starting so that users may more
     // easily parse which scenario the messages pertain to.
-    LoggerFactory::logNewScenarioStarting( overrideName );
+    LoggerFactory::getInstance()->logNewScenarioStarting( overrideName );
 
     // Print data read in time.
     mainLog.setLevel( ILogger::DEBUG );

cvs/objects/emissions/include/aghg.h

@@ -56,13 +56,13 @@
 #include "util/base/include/value.h"
 #include "util/base/include/time_vector.h"
 #include "util/base/include/data_definition_util.h"
+#include "marketplace/include/cached_market.h"
 
 // Forward declarations
 class IInfo;
 class IOutput;
 class ICaptureComponent;
 class IInput;
-class CachedMarket;
 
 // Need to forward declare the subclasses as well.
 class CO2Emissions;
@@ -208,7 +208,7 @@ class AGHG: public INamed, public IVisitable, private boost::noncopyable
 
     //! Pre-located market which has been cached from the marketplace to get the price
     //! of this ghg and add demands to the market.
-    std::unique_ptr<CachedMarket> mCachedMarket;
+    CachedMarket mCachedMarket;
 
     /*!
      * \brief XML debug output stream for derived classes

cvs/objects/emissions/source/aghg.cpp

@@ -123,7 +123,13 @@ void AGHG::completeInit( const string& aRegionName, const string& aSectorName,
  * \param aPeriod Model period.
  */
 void AGHG::initCalc( const string& aRegionName, const IInfo* aLocalInfo, const int aPeriod ) {
-    mCachedMarket = scenario->getMarketplace()->locateMarket( getName(), aRegionName, aPeriod );
+    // Ideally we only need to locate the market once, however during completeInit
+    // all markets may have not yet been set up.  So, instead we avoid re-lookups
+    // if the market has been found.  Unfortunately, this means if the market will
+    // never be found we will continue to try to look it up each model period.
+    if(!mCachedMarket.hasLocatedMarket()) {
+        mCachedMarket = scenario->getMarketplace()->locateMarket( getName(), aRegionName );
+    }
 }
 
 /*!
@@ -133,9 +139,9 @@ void AGHG::initCalc( const string& aRegionName, const IInfo* aLocalInfo, const i
  */
 void AGHG::addEmissionsToMarket( const string& aRegionName, const int aPeriod ){
     // set emissions as demand side of gas market
-    mCachedMarket->addToDemand( getName(), aRegionName,
-                                mEmissions[ aPeriod ],
-                                aPeriod, false );
+    mCachedMarket.addToDemand( getName(), aRegionName,
+                               mEmissions[ aPeriod ],
+                               aPeriod, false );
 }
 
 /*! Second Method: Convert GHG tax and any storage costs into energy units using
@@ -157,7 +163,7 @@ double AGHG::getGHGValue( const IInput* aInput, const string& aRegionName,
                           const int aPeriod ) const
 {
     // Determine if there is a tax.
-    double ghgTax = mCachedMarket->getPrice( getName(), aRegionName, aPeriod, false );
+    double ghgTax = mCachedMarket.getPrice( getName(), aRegionName, aPeriod, false );
     if( ghgTax == Marketplace::NO_MARKET_PRICE ){
         ghgTax = 0;
     }
@@ -195,7 +201,7 @@ double AGHG::getGHGValue( const IOutput* aOutput, const string& aRegionName,
                           const int aPeriod ) const
 {
     // Determine if there is a tax.
-    double ghgTax = mCachedMarket->getPrice( getName(), aRegionName, aPeriod, false );
+    double ghgTax = mCachedMarket.getPrice( getName(), aRegionName, aPeriod, false );
     if( ghgTax == Marketplace::NO_MARKET_PRICE ){
         ghgTax = 0;
     }

cvs/objects/emissions/source/co2_emissions.cpp

@@ -155,7 +155,7 @@ double CO2Emissions::getGHGValue( const std::string& aRegionName,
     double removeFraction = aSequestrationDevice ? aSequestrationDevice->getRemoveFraction( getName() ) : 0;
 
     // Get the greenhouse gas tax from the marketplace.
-    double GHGTax = mCachedMarket->getPrice( getName(), aRegionName, aPeriod, false );
+    double GHGTax = mCachedMarket.getPrice( getName(), aRegionName, aPeriod, false );
 
     if( GHGTax == Marketplace::NO_MARKET_PRICE ){
         GHGTax = 0;

cvs/objects/emissions/source/nonco2_emissions.cpp

@@ -255,7 +255,7 @@ double NonCO2Emissions::getGHGValue( const string& aRegionName,
     // Conversion from teragrams (Tg=MT) of X per EJ to metric tons of X per GJ
     const double CVRT_Tg_per_EJ_to_Tonne_per_GJ = 1e-3;
 
-    double GHGTax = mCachedMarket->getPrice( getName(), aRegionName, aPeriod, false );
+    double GHGTax = mCachedMarket.getPrice( getName(), aRegionName, aPeriod, false );
     if( GHGTax == Marketplace::NO_MARKET_PRICE ){
         return 0;
     }

cvs/objects/functions/include/energy_input.h

@@ -50,10 +50,10 @@
 #include "functions/include/minicam_input.h"
 #include "util/base/include/value.h"
 #include "util/base/include/time_vector.h"
+#include "marketplace/include/cached_market.h"
 
 class Tabs;
 class ICoefficient;
-class CachedMarket;
 
 /*! 
  * \ingroup Objects
@@ -205,7 +205,7 @@ class EnergyInput: public MiniCAMInput
 
     //! A pre-located market which has been cahced from the marketplace to get
     //! the price and add demands to.
-    std::unique_ptr<CachedMarket> mCachedMarket;
+    CachedMarket mCachedMarket;
 
 private:
     const static std::string XML_REPORTING_NAME; //!< tag name for reporting xml db 

cvs/objects/functions/source/energy_input.cpp

@@ -244,7 +244,13 @@ void EnergyInput::initCalc( const string& aRegionName,
         mAdjustedCoefficients[ aPeriod ] = 1;
     }
 
-    mCachedMarket = scenario->getMarketplace()->locateMarket( mName, mMarketName, aPeriod );
+    // Ideally we only need to locate the market once, however during completeInit
+    // all markets may have not yet been set up.  So, instead we avoid re-lookups
+    // if the market has been found.  Unfortunately, this means if the market will
+    // never be found we will continue to try to look it up each model period.
+    if(!mCachedMarket.hasLocatedMarket()) {
+        mCachedMarket = scenario->getMarketplace()->locateMarket( mName, mMarketName );
+    }
 }
 
 /*! \brief Initialize the type flags.
@@ -302,9 +308,9 @@ void EnergyInput::setPhysicalDemand( double aPhysicalDemand,
                                      const int aPeriod )
 {
     mPhysicalDemand[ aPeriod ].set( aPhysicalDemand );
-    mCachedMarket->addToDemand( mName, mMarketName,
-                                       mPhysicalDemand[ aPeriod ],
-                                       aPeriod, true );
+    mCachedMarket.addToDemand( mName, mMarketName,
+                               mPhysicalDemand[ aPeriod ],
+                               aPeriod, true );
 }
 
 double EnergyInput::getCoefficient( const int aPeriod ) const {
@@ -328,7 +334,7 @@ double EnergyInput::getPrice( const string& aRegionName,
                               const int aPeriod ) const
 {
     return mPriceUnitConversionFactor *
-        mCachedMarket->getPrice( mName, mMarketName, aPeriod );
+        mCachedMarket.getPrice( mName, mMarketName, aPeriod );
 }
 
 void EnergyInput::setPrice( const string& aRegionName,

cvs/objects/functions/source/food_demand_function.cpp

@@ -139,13 +139,6 @@ double FoodDemandFunction::calcDemand( InputSet& aInput, double income, const st
         for( size_t j = 0; j < aInput.size(); ++j ) {
             currDemand *= pow( adjPricesCapped[j], foodInputs[i]->calcPriceExponent( foodInputs[j], adjIncome, aRegionName, aPeriod ) );
         }
-        if( adjPrices[i] < adjPricesCapped[i] ) {
-            // we have been sent negative prices, since we have capped prices
-            // in the demand calculations above we need to apply some penalty
-            // to send a signal to the solver such that the more negative a price
-            // becomes, the higher the demand
-            currDemand = SectorUtils::adjustDemandForNegativePrice( currDemand, adjPrices[i] );
-        }
         demands[i] = currDemand;
         // the demand for materials is just the residual of the food demand:
         // q_m = x - SUM_i(w_i * q_i)
@@ -174,6 +167,13 @@ double FoodDemandFunction::calcDemand( InputSet& aInput, double income, const st
     // point, the inputs will take care of converting to Pcal / year as
     // is expected in the supply sectors.
     for( size_t i = 0; i < aInput.size(); ++i ) {
+        if( adjPrices[i] < adjPricesCapped[i] ) {
+            // we have been sent negative prices, since we have capped prices
+            // in the demand calculations above we need to apply some penalty
+            // to send a signal to the solver such that the more negative a price
+            // becomes, the higher the demand
+            demands[i] = SectorUtils::adjustDemandForNegativePrice( demands[i], adjPrices[i] );
+        }
         foodInputs[i]->setPhysicalDemand( demands[i], aRegionName, aPeriod );
         foodInputs[i]->setActualShare( alphaActual[i], aRegionName, aPeriod );
     }

cvs/objects/functions/source/function_utils.cpp

@@ -128,7 +128,7 @@ double FunctionUtils::DEFLATOR_1975_PER_DEFLATOR_2005( void )
 //! Function to return 1990 to 1975 US $ deflator.
 double FunctionUtils::DEFLATOR_1990_PER_DEFLATOR_1975( void )
 {
-    return 2.212; // Ratio of 1990/1975 deflators
+    return 2.129; // Ratio of 1990/1975 deflators
 }
 
 //! Function to return kWh to GJ conversion.

cvs/objects/functions/source/nested_ces_production_function_macro.cpp

@@ -318,25 +318,23 @@ double NestedCESProductionFunctionMacro::calcGrossOutput( const string& aRegionN
         double grossOutput = aNationalAccount->getAccountValue(NationalAccount::GROSS_OUTPUT);
 
         // calcualte value shares
-        // Note: in principle the energy value and quantity could be different but given
-        // energy the Q is just a service index weighted by prices these are indeed the same
-        double energyV = energyQ;
         double laborV = aNationalAccount->getAccountValue(NationalAccount::LABOR_WAGES);
+        double capitalV = aNationalAccount->getAccountValue(NationalAccount::CAPITAL_VALUE);
 
-        double shareE = energyV / grossOutput;
+        double shareK = capitalV / grossOutput;
         double shareL = laborV / grossOutput;
-        double shareK = (1.0 - shareE - shareL);
+        double shareE = (1.0 - shareK - shareL);
         // Given the energy value is being pulled out of GCAM without adjustment
-        // there is a chance for inconsistency and send the remaining value (which is
-        // all attributed to capital) to <= zero.  Instead we will enforce a minimum
-        // capital share and adjust both labor and energy to make up for the shortfall
-        const double MIN_CAPITAL_SHARE = 0.05;
-        if(shareK < MIN_CAPITAL_SHARE) {
-            double shareAdj = MIN_CAPITAL_SHARE - shareK;
+        // there is a chance for inconsistency and send the remaining value to <= zero.
+        // Instead we will enforce a minimum energy share and adjust both labor and capital
+        // to make up for the shortfall
+        const double MIN_SLACK_SHARE = 0.05;
+        if(shareE < MIN_SLACK_SHARE) {
+            double shareAdj = (1.0 - MIN_SLACK_SHARE) / (shareK + shareL);
             // adjust labor and energy uniformily to meet the minimum threshold
-            shareE -= shareAdj / 2.0;
-            shareL -= shareAdj / 2.0;
-            shareK = MIN_CAPITAL_SHARE;
+            shareK *= shareAdj;
+            shareL *= shareAdj;
+            shareE = MIN_SLACK_SHARE;
         }
 
         // calibrate and store scalers

cvs/objects/java/source/WriteLocalBaseXDB.java

@@ -54,34 +54,34 @@ public class WriteLocalBaseXDB implements Runnable {
     /**
      * The database context needed to run commands on the DB.
      */
-    private Context mContext = null;
+    protected Context mContext = null;
 
     /**
      * The thread on which writing to the DB will take place.
      */
-    private final Thread mWorkerThread = new Thread( this );
+    protected final Thread mWorkerThread = new Thread( this );
 
     /**
      * The command which executes adding the XML to the database.
      * We keep a reference here in case we need to cancel the
      * command incase something went wrong.
      */
-    private Add mAddCommand = null;
+    protected Add mAddCommand = null;
 
     /**
      * The stream that will transfer the XML read from GCAM and write it to the DB.
      */
-    private final PipedInputStream mWriteToDBStream = new PipedInputStream( XMLDBDriver.BUFFER_SIZE );
+    protected final PipedInputStream mWriteToDBStream = new PipedInputStream( XMLDBDriver.BUFFER_SIZE );
 
     /**
      * The location of the database to write the XML to.
      */
-    private final String mDBLocation;
+    protected final String mDBLocation;
 
     /**
      * A unique name to call the document to be added into the DB.
      */
-    private final String mDocName;
+    protected final String mDocName;
 
     /**
      * Constructor which will open the DB and get ready to receive XML to put
@@ -140,7 +140,7 @@ public Context getContext() {
      *                    attempting to write to a DB which appears to be open. A
      *                    negative value indicates to wait indefinately.
      */
-    private void openDB( final boolean aInMemoryDB, final int aOpenDBWait ) throws Exception {
+    protected void openDB( final boolean aInMemoryDB, final int aOpenDBWait ) throws Exception {
         // We need to seperate the path to the DB and the container name (last name in the path)
         File dbLocationFile = new File( mDBLocation ).getAbsoluteFile();
         // The path may be a relative path so we must convert it to absolute here.