Pull request #2: debug CAL_7 and split synthetic tests into pieces to…

… allow slow and fast integration testing Merge in CEFL/lisflood-calibration from test_synthetic to develop * commit 'e2f22813f6b05723011282e3d95650e128c817fe': debug CAL_7 and split synthetic tests into pieces to allow slow and fast integration testing
ec-jrc · Jun 3, 2021 · 55dcb5a · 55dcb5a
2 parents dd507b4 + e2f2281
commit 55dcb5a
Show file tree

Hide file tree

Showing 16 changed files with 230 additions and 268 deletions.
diff --git a/bin/CAL_7_PERFORM_CAL.py b/bin/CAL_7_PERFORM_CAL.py
@@ -2,14 +2,15 @@
 """Please refer to quick_guide.pdf for usage instructions"""
 import os
 import sys
+import argparse
 import numpy as np
 import pandas
 from configparser import ConfigParser # Python 3.8
 import glob
 import subprocess
 import traceback
 
-from liscal import templates, calibration, config, subcatchment, objective
+from liscal import templates, calibration, config, subcatchment, objective, hydro_model
 
 
 def calibrate_subcatchment(cfg, obsid, station_data):
@@ -44,43 +45,36 @@ def calibrate_subcatchment(cfg, obsid, station_data):
     hydro_model.generate_outlet_streamflow(cfg, subcatch, lis_template)
 
 
-def calibrate_system(args):
-    ########################################################################
-    #   Read settings file
-    ########################################################################
-    if len(args) == 0:
-        print(args)
-        settings_file = os.path.normpath(sys.argv[1])
-        subcatchments_list = os.path.normpath(sys.argv[2])
-    else:
-        print(sys.argv)
-        settings_file = os.path.normpath(args[0])
-        subcatchments_list = os.path.normpath(args[1])
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('settings_file', help='Calibration settings file')
+    parser.add_argument('stations', help='List of stations to process')
+    args = parser.parse_args()
+
+    settings_file = args.settings_file
+    stations_list = args.stations
 
     cfg = config.Config(settings_file)
 
     # Read full list of stations, index is obsid
     print(">> Reading Qmeta2.csv file...")
-    stations = pandas.read_csv(os.path.join(cfg.path_result, "Qmeta2.csv"), sep=",", index_col=0)
+    stations_meta = pandas.read_csv(cfg.Qmeta_csv, sep=",", index_col=0)
 
     # Read list of stations we want to calibrate
-    subcatchments = pandas.read_csv(subcatchments_list, sep=",", header=None)
-    obsid_list = subcatchments.loc[:, 0]
+    stations = pandas.read_csv(stations_list, sep=",", header=None)
+    obsid_list = stations.loc[:, 0]
 
     ########################################################################
     #   Loop through subcatchments and perform calibration
     ########################################################################
     for obsid in obsid_list:
 
         try:
-            station_data = stations.loc[obsid]
+            station_data = stations_meta.loc[obsid]
         except KeyError as e:
             raise Exception('Station {} not found in stations file'.format(obsid))
 
         calibrate_subcatchment(cfg, obsid, station_data)
 
     print("==================== END ====================")
-
-
-if __name__ == '__main__':
-    calibrate_system()
diff --git a/integration/benchmark.py b/integration/benchmark.py
@@ -0,0 +1,56 @@
+#!/bin/python3
+import sys
+import os
+from os import path
+import re
+import argparse
+import pandas
+import numpy as np
+from datetime import datetime, timedelta
+
+from liscal import hydro_model, templates, config, subcatchment, utils
+
+
+def deleteOutput(subcatch_dir):
+    ret, res = utils.run_cmd("rm -f {}/settings*.xml".format(subcatch_dir))
+    ret, res = utils.run_cmd("rm -rf {}/out".format(subcatch_dir))
+    ret, res = utils.run_cmd("rm -rf {}/*.csv".format(subcatch_dir))
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('obsid', help='Station obsid')
+    parser.add_argument('settings', help='Settings file')
+    args = parser.parse_args()
+
+    print('  - obsid: {}'.format(args.obsid))
+    print('  - settings file: {}'.format(args.settings))
+    obsid = int(args.obsid)
+    cfg = config.Config(args.settings)
+
+    print(">> Reading stations.csv file...")
+    stations = pandas.read_csv(cfg.Qmeta_csv, sep=",", index_col=0)
+    try:
+        station_data = stations.loc[obsid]
+    except KeyError as e:
+        print(stations)
+        raise Exception('Station {} not found in stations file'.format(obsid))
+
+    # hack shorter period
+    assert station_data.loc['Cal_Start'] == cfg.forcing_start.strftime('%d/%m/%Y %H:%M')
+    assert station_data.loc['Cal_End'] == cfg.forcing_end.strftime('%d/%m/%Y %H:%M')
+
+    print("=================== "+str(obsid)+" ====================")
+    subcatch = subcatchment.SubCatchment(cfg, obsid, station_data)
+    out_file = os.path.join(subcatch.path, 'out', 'convergenceTester.csv')
+    if os.path.exists(out_file):
+        deleteOutput(subcatch.path)
+        os.makedirs(subcatch.path_out, exist_ok=True)
+
+    # create object to create lisflood settings file
+    lis_template = templates.LisfloodSettingsTemplate(cfg, subcatch)
+
+    # first generate synthetic observations
+    param_target = 0.5*np.ones(len(cfg.param_ranges))
+    hydro_model.generate_benchmark(cfg, subcatch, lis_template, param_target, out_file)
diff --git a/integration/check_kge.py b/integration/check_kge.py
@@ -0,0 +1,29 @@
+import argparse
+import pandas as pd
+
+
+def check_kge(kge_file, target1, target2, tol):
+
+    df = pd.read_csv(kge_file)
+    kge_max = float(df['effmax_R'][-1:])
+    check = abs(kge_max - target1) < tol and kge_max > target2
+
+    if check:
+        print('Yay! KGE target reached!')
+    else:
+        raise Exception("Target not reached! abs({} - {}) = {} is > {} or {} < {}".format(kge_max, target1, abs(kge_max-target1), tol, kge_max, target2))
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('kge_file', help='KGE front history file')
+    parser.add_argument('tol', help='KGE history file')
+    args = parser.parse_args()
+
+    tol = float(args.tol)
+    print('KGE file: {}'.format(args.kge_file))
+    print('Tolerance is {}'.format(tol))
+
+    # check KGE
+    check_kge(args.kge_file, target1=1., target2=0.99, tol=2*tol)
diff --git a/integration/settings_fast.txt b/integration/settings_fast.txt
@@ -0,0 +1,30 @@
+[DEFAULT]
+Root = ROOT
+Src = SRC
+ObservationsStart = 1/1/1990 00:00
+ObservationsEnd = 01/07/2018 00:00
+ForcingStart = 31/12/2016 06:00
+ForcingEnd = 31/12/2017 06:00
+WarmupDays = 30
+calibrationFreq = 6-hourly
+fastDebug = 0
+
+[CSV]
+Qmeta = STATIONS
+Qtss = OBS
+
+[Path]
+Result = %(Root)s/result/
+SubCatchmentPath = %(Root)s/catchments/
+ParamRanges = %(Src)s/tests/data/ParamRanges_LISFLOOD.csv
+
+[Templates]
+LISFLOODSettings = %(Src)s/templates/settings_LF_CUT.xml
+
+[DEAP]
+numCPUs = NCPUS
+minGen = 1
+maxGen = 1
+mu = 2
+lambda_ = 2
+pop = 2
diff --git a/integration/settings_slow.txt b/integration/settings_slow.txt
@@ -0,0 +1,30 @@
+[DEFAULT]
+Root = ROOT
+Src = SRC
+ObservationsStart = 1/1/1990 00:00
+ObservationsEnd = 01/07/2018 00:00
+ForcingStart = 31/12/2016 06:00
+ForcingEnd = 31/12/2017 06:00
+WarmupDays = 30
+calibrationFreq = 6-hourly
+fastDebug = 0
+
+[CSV]
+Qmeta = STATIONS
+Qtss = OBS
+
+[Path]
+Result = %(Root)s/result/
+SubCatchmentPath = %(Root)s/catchments/
+ParamRanges = %(Src)s/tests/data/ParamRanges_LISFLOOD.csv
+
+[Templates]
+LISFLOODSettings = %(Src)s/templates/settings_LF_CUT.xml
+
+[DEAP]
+numCPUs = NCPUS
+minGen = 6
+maxGen = 16
+mu = 18
+lambda_ = 36
+pop = 72
diff --git a/integration/settings_tests.txt b/integration/settings_tests.txt
diff --git a/integration/stations.csv b/integration/stations.csv
@@ -1,15 +1,15 @@
 ObsID,StationName,Provider ID,Country code,StationLat,StationLon,Height,Height Units,DrainingArea.km2.Provider,Catchment Area Units,Added Date,River,Catchment,EC_Catchments,Calibration ID,DrainingArea.km2.LDD,LisfloodX,LisfloodY,RealTime,PostProcess,FixedRepPoint,HasWL,HasD,Alive D,StartDate_hist_24,EndDate_hist_24,StartDate_nrt_24,EndDate_nrt_24,StartDate_hist_6,EndDate_hist_6,StartDate_nrt_6,EndDate_nrt_6,DiffDays_hist_24,DiffDays_nrt_24,DiffDays_hist_6,DiffDays_nrt_6,cal_hist_24h,cal_nrt_24h,cal_hist_6h,cal_nrt_6h,CAL_TYPE,Notes,EC_calib,Dam/Lake,EnoughQdata,Val_Start,Val_End,Cal_Start,Cal_End,CatchmentArea,SamplingFrequency
-380,Borgotaro,1016,IT,44.514057,9.84219,354,m,-,km2,18/09/2012 19:00,Taro,Po,Po,-,400,4307500,2377500,False,FALSE,True,1,0,1,21/09/2006 00:00,01/07/2008 00:00,09/05/2012 00:00,08/11/2017 00:00,20/09/2006 18:00,01/07/2008 00:00,08/05/2012 12:00,07/11/2017 12:00,649,2009,649,2009,False,True,False,True,NRT_6h,-,T,-,1.0,08/05/2012 12:00,02/01/2012 06:00,02/01/2009 06:00,07/11/2017 12:00,16.0,2.0
+380,Borgotaro,1016,IT,44.514057,9.84219,354,m,-,km2,18/09/2012 19:00,Taro,Po,Po,-,400,4307500,2377500,False,FALSE,True,1,0,1,21/09/2006 00:00,01/07/2008 00:00,09/05/2012 00:00,08/11/2017 00:00,20/09/2006 18:00,01/07/2008 00:00,08/05/2012 12:00,07/11/2017 12:00,649,2009,649,2009,False,True,False,True,NRT_6h,-,T,-,1.0,08/05/2012 12:00,02/01/2012 06:00,31/12/2016 06:00,31/12/2017 06:00,16.0,2.0
 2733,Raskov Morava,1062,CZ,50.041111,16.912778,-,-,9809.42,km2,13/04/2018 06:41,Morava,"Morava, above March",Danube,-,425,4812500,3012500,False,-,True,0,1,0,02/11/1993 00:00,01/11/2002 00:00,-,-,-,-,-,-,3286,-,-,-,True,False,False,False,HIST_24h,Wrong area from provider,T,-,1.0,02/11/1993 00:00,02/01/1998 06:00,02/01/1995 06:00,01/11/2002 00:00,17.0,1.0
-2730,Bejanovo ,1062,BG,43.2331,24.4089,-,-,3323.31,km2,13/04/2018 06:41,Kamenka,Vit,Danube,-,450,5487500,2347500,False,-,True,0,1,0,02/01/1991 00:00,01/01/2004 00:00,-,-,-,-,-,-,4747,-,-,-,True,False,False,False,HIST_24h,moved on affluent,T,-,1.0,02/01/1991 00:00,02/01/1997 06:00,02/01/1994 06:00,01/01/2004 00:00,18.0,1.0
-1976,KEMPTEN,1062,DE,47.73,10.31,-,-,955,km2,19/03/2018 16:00,Iller,Danube,Danube,C356,950,4347500,2732500,False,-,True,0,1,0,02/11/1900 00:00,01/01/2009 00:00,-,-,-,-,-,-,39507,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,03/01/1990 00:00,02/01/1999 06:00,02/01/1996 06:00,01/01/2009 00:00,38.0,2.0
-2910,VILLAFRANCA PELLICE,1062,IT,44.79393416,7.495516929,-,-,998,km2,08/06/2018 11:14,Pellice,Po,Po,C685,1000,4122500,2412500,False,-,True,0,0,0,02/01/2002 00:00,01/01/2009 00:00,-,-,-,-,-,-,2556,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2002 00:00,02/01/2004 06:00,02/01/2001 06:00,01/01/2009 00:00,40.0,2.0
-287,Chmelnica,1018,SK,49.28917681,20.7301459,507.37,m,1262.41,km2,20/09/2012 19:00,Poprad,Vistula,Vistula,-,1325,5097500,2967500,True,TRUE,True,1,1,1,02/11/1930 00:00,01/01/2017 00:00,27/03/2012 00:00,31/08/2018 00:00,01/01/2002 06:00,01/01/2017 00:00,26/03/2012 18:00,30/08/2018 06:00,31472,2348,5478,2347,True,True,True,True,HIST_6h,-,T,-,1.0,01/01/2002 06:00,02/01/2002 06:00,02/01/1999 06:00,01/01/2017 00:00,53.0,2.0
-2824,Fossano,1062,IT,44.52795852,7.695741194,-,-,1850,km2,08/06/2018 11:14,Stura Demonte,Po,Po,C472,1400,4142500,2387500,False,-,True,0,0,0,02/01/2000 00:00,31/12/2008 00:00,-,-,-,-,-,-,3286,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2000 00:00,02/01/2004 06:00,02/01/2001 06:00,31/12/2008 00:00,56.0,2.0
-428,San Secondo,1016,IT,44.916891,10.254303,42,m,-,km2,18/09/2012 19:00,Taro,Po,Po,-,1450,4342500,2422500,False,FALSE,True,1,0,1,-,-,12/06/2012 00:00,08/11/2017 00:00,-,-,11/06/2012 18:00,07/11/2017 12:00,-,1975,-,1974,False,True,False,True,NRT_6h,-,T,-,1.0,11/06/2012 18:00,02/01/2012 06:00,02/01/2009 06:00,07/11/2017 12:00,58.0,1.0
-2822,Farigliano,1062,IT,44.48688901,7.885693666,-,-,1150,km2,08/06/2018 11:14,Tanaro,Po,Po,C466,1450,4152500,2372500,False,-,True,0,0,0,02/01/2003 00:00,31/12/2008 00:00,-,-,-,-,-,-,2190,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2003 00:00,02/01/2004 06:00,02/01/2001 06:00,31/12/2008 00:00,58.0,2.0
-892,Wiblingen,1025,DE,48.3708,9.9874,46824,cm,2040.21,km2,18/09/2012 19:00,Iller,Danube,Danube,C053,2300,4322500,2802500,True,FALSE,True,1,1,1,02/01/1991 00:00,01/01/2007 00:00,14/01/2013 00:00,31/08/2018 00:00,-,-,14/01/2013 00:00,30/08/2018 12:00,5843,2055,-,2054,True,True,False,True,NRT_6h,-,T,-,1.0,14/01/2013 00:00,02/01/2013 06:00,02/01/2010 06:00,31/12/2017 06:00,92.0,1.0
-2787,Carignano,1062,IT,44.88839372,7.68094123,-,-,3775,km2,08/06/2018 11:14,Po,Po,Po,C130,3825,4137500,2417500,False,-,True,0,0,0,02/01/2000 00:00,31/12/2006 00:00,-,-,-,-,-,-,2555,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2000 00:00,02/01/2002 06:00,02/01/1999 06:00,31/12/2006 00:00,153.0,1.0
-2823,Asti,1062,IT,44.89944053,8.249440273,-,-,4800,km2,08/06/2018 11:14,Tanaro,Po,Po,C471,4475,4187500,2417500,False,-,True,0,0,0,02/07/2000 00:00,31/12/2008 00:00,-,-,-,-,-,-,3104,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2004 00:00,02/01/2004 06:00,02/01/2001 06:00,31/12/2008 00:00,179.0,1.0
-817,Kormend,1017,HU,47.0078,16.6184,184.15,-,-,-,20/09/2012 19:00,Raba,Danube,Danube,-,4775,4822500,2677500,True,FALSE,True,1,1,1,02/01/2015 00:00,01/01/2017 00:00,07/09/2009 00:00,29/08/2018 00:00,01/01/2015 06:00,01/01/2017 00:00,-,-,730,3278,730,-,False,True,False,False,NRT_24h,-,T,-,1.0,07/09/2009 00:00,02/01/2013 06:00,02/01/2010 06:00,31/12/2017 00:00,191.0,1.0
-492,ŻABNO,1024,PL,50.13194444,20.86361111,172.427,mamsl,6740.66,Km2,20/09/2012 19:00,Dunajec,Vistula,Vistula,C225,6900,5097500,3052500,False,TRUE,True,0,0,0,02/01/1995 00:00,01/11/2016 00:00,18/03/2018 00:00,31/08/2018 00:00,-,-,17/03/2018 18:00,30/08/2018 12:00,7974,166,-,165,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/1995 00:00,02/01/2003 06:00,02/01/2000 06:00,01/11/2016 00:00,276.0,1.0
+2730,Bejanovo ,1062,BG,43.2331,24.4089,-,-,3323.31,km2,13/04/2018 06:41,Kamenka,Vit,Danube,-,450,5487500,2347500,False,-,True,0,1,0,02/01/1991 00:00,01/01/2004 00:00,-,-,-,-,-,-,4747,-,-,-,True,False,False,False,HIST_24h,moved on affluent,T,-,1.0,02/01/1991 00:00,02/01/1997 06:00,31/12/2016 06:00,31/12/2017 06:00,18.0,1.0
+1976,KEMPTEN,1062,DE,47.73,10.31,-,-,955,km2,19/03/2018 16:00,Iller,Danube,Danube,C356,950,4347500,2732500,False,-,True,0,1,0,02/11/1900 00:00,01/01/2009 00:00,-,-,-,-,-,-,39507,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,03/01/1990 00:00,02/01/1999 06:00,31/12/2016 06:00,31/12/2017 06:00,38.0,2.0
+2910,VILLAFRANCA PELLICE,1062,IT,44.79393416,7.495516929,-,-,998,km2,08/06/2018 11:14,Pellice,Po,Po,C685,1000,4122500,2412500,False,-,True,0,0,0,02/01/2002 00:00,01/01/2009 00:00,-,-,-,-,-,-,2556,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2002 00:00,02/01/2004 06:00,31/12/2016 06:00,31/12/2017 06:00,40.0,2.0
+287,Chmelnica,1018,SK,49.28917681,20.7301459,507.37,m,1262.41,km2,20/09/2012 19:00,Poprad,Vistula,Vistula,-,1325,5097500,2967500,True,TRUE,True,1,1,1,02/11/1930 00:00,01/01/2017 00:00,27/03/2012 00:00,31/08/2018 00:00,01/01/2002 06:00,01/01/2017 00:00,26/03/2012 18:00,30/08/2018 06:00,31472,2348,5478,2347,True,True,True,True,HIST_6h,-,T,-,1.0,01/01/2002 06:00,02/01/2002 06:00,31/12/2016 06:00,31/12/2017 06:00,53.0,2.0
+2824,Fossano,1062,IT,44.52795852,7.695741194,-,-,1850,km2,08/06/2018 11:14,Stura Demonte,Po,Po,C472,1400,4142500,2387500,False,-,True,0,0,0,02/01/2000 00:00,31/12/2008 00:00,-,-,-,-,-,-,3286,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2000 00:00,02/01/2004 06:00,31/12/2016 06:00,31/12/2017 06:00,56.0,2.0
+428,San Secondo,1016,IT,44.916891,10.254303,42,m,-,km2,18/09/2012 19:00,Taro,Po,Po,-,1450,4342500,2422500,False,FALSE,True,1,0,1,-,-,12/06/2012 00:00,08/11/2017 00:00,-,-,11/06/2012 18:00,07/11/2017 12:00,-,1975,-,1974,False,True,False,True,NRT_6h,-,T,-,1.0,11/06/2012 18:00,02/01/2012 06:00,31/12/2016 06:00,31/12/2017 06:00,58.0,1.0
+2822,Farigliano,1062,IT,44.48688901,7.885693666,-,-,1150,km2,08/06/2018 11:14,Tanaro,Po,Po,C466,1450,4152500,2372500,False,-,True,0,0,0,02/01/2003 00:00,31/12/2008 00:00,-,-,-,-,-,-,2190,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2003 00:00,02/01/2004 06:00,31/12/2016 06:00,31/12/2017 06:00,58.0,2.0
+892,Wiblingen,1025,DE,48.3708,9.9874,46824,cm,2040.21,km2,18/09/2012 19:00,Iller,Danube,Danube,C053,2300,4322500,2802500,True,FALSE,True,1,1,1,02/01/1991 00:00,01/01/2007 00:00,14/01/2013 00:00,31/08/2018 00:00,-,-,14/01/2013 00:00,30/08/2018 12:00,5843,2055,-,2054,True,True,False,True,NRT_6h,-,T,-,1.0,14/01/2013 00:00,02/01/2013 06:00,31/12/2016 06:00,31/12/2017 06:00,92.0,1.0
+2787,Carignano,1062,IT,44.88839372,7.68094123,-,-,3775,km2,08/06/2018 11:14,Po,Po,Po,C130,3825,4137500,2417500,False,-,True,0,0,0,02/01/2000 00:00,31/12/2006 00:00,-,-,-,-,-,-,2555,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2000 00:00,02/01/2002 06:00,31/12/2016 06:00,31/12/2017 06:00,153.0,1.0
+2823,Asti,1062,IT,44.89944053,8.249440273,-,-,4800,km2,08/06/2018 11:14,Tanaro,Po,Po,C471,4475,4187500,2417500,False,-,True,0,0,0,02/07/2000 00:00,31/12/2008 00:00,-,-,-,-,-,-,3104,-,-,-,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/2004 00:00,02/01/2004 06:00,31/12/2016 06:00,31/12/2017 06:00,179.0,1.0
+817,Kormend,1017,HU,47.0078,16.6184,184.15,-,-,-,20/09/2012 19:00,Raba,Danube,Danube,-,4775,4822500,2677500,True,FALSE,True,1,1,1,02/01/2015 00:00,01/01/2017 00:00,07/09/2009 00:00,29/08/2018 00:00,01/01/2015 06:00,01/01/2017 00:00,-,-,730,3278,730,-,False,True,False,False,NRT_24h,-,T,-,1.0,07/09/2009 00:00,02/01/2013 06:00,31/12/2016 06:00,31/12/2017 06:00,191.0,1.0
+492,ŻABNO,1024,PL,50.13194444,20.86361111,172.427,mamsl,6740.66,Km2,20/09/2012 19:00,Dunajec,Vistula,Vistula,C225,6900,5097500,3052500,False,TRUE,True,0,0,0,02/01/1995 00:00,01/11/2016 00:00,18/03/2018 00:00,31/08/2018 00:00,-,-,17/03/2018 18:00,30/08/2018 12:00,7974,166,-,165,True,False,False,False,HIST_24h,-,T,-,1.0,02/01/1995 00:00,02/01/2003 06:00,31/12/2016 06:00,31/12/2017 06:00,276.0,1.0