Merge pull request #95 from nismod/water-supply

Wrap water supply model

.gitignore

@@ -143,3 +143,7 @@ data/*
 tmp/*
 results/*
 provision/cache/*
+
+# Water supply
+models/water_supply/nodal
+models/water_supply/exe

README.md

@@ -211,6 +211,7 @@ bash ./provision/get_data_digital_comms.sh .
 bash ./provision/get_data_energy_demand.sh .
 bash ./provision/get_data_energy_supply.sh .
 bash ./provision/get_data_transport.sh .
+bash ./provision/get_data_water_supply.sh .
 ```
 
 ### Install models
@@ -222,6 +223,7 @@ bash ./provision/install_digital_comms.sh .
 bash ./provision/install_energy_demand.sh .
 bash ./provision/install_energy_supply.sh . ./path/to/xpress_install_dir
 bash ./provision/install_transport.sh .
+bash ./provision/install_water_supply.sh .
 ```
 
 Run post-install scripts:
@@ -230,6 +232,8 @@ Run post-install scripts:
 energy_demand minimal_setup -d ./models/energy_demand/wrapperconfig.ini
 ```
 
+Note that `install_water_supply.sh` clones a repository that is currently private: setting up the water supply model may require requesting access to the repository.
+
 
 ## Running NISMOD on a virtual machine
 

config/dimensions/water_supply_arcs.yml

@@ -0,0 +1,3 @@
+name: water_supply_arcs
+description: Names of water supply arcs
+elements: water_supply/water_supply_arcs.csv

config/dimensions/water_supply_days_into_year.yml

@@ -0,0 +1,3 @@
+name: water_supply_days_into_year
+description: Days into year, 1, ..., 365
+elements: water_supply/days_into_year.csv

config/dimensions/water_supply_reservoirs.yml

@@ -0,0 +1,3 @@
+name: water_supply_reservoirs
+description: Names of water supply reservoirs
+elements: water_supply/water_supply_reservoirs.csv

config/model_runs/water_supply_test.yml

@@ -0,0 +1,9 @@
+description: ''
+name: water_supply_test
+narratives: {}
+scenarios: {}
+sos_model: water_supply_only
+stamp: '2019-03-01T00:00:00.000Z'
+timesteps:
+- 1999
+strategies: []

config/scenarios/water_supply.yml

@@ -0,0 +1,14 @@
+name: water_supply
+description: Daily water supply
+provides:
+  - name: water_supply_arcs
+    description: ''
+    dims:
+    - water_supply_arcs
+    dtype: float
+    unit: DontKnow
+variants:
+  - name: water_supply_baseline
+    description: Water supply
+    data:
+      something: tbc

config/sector_models/water_supply.yml

@@ -0,0 +1,21 @@
+classname: WaterWrapper
+description: Water supply model
+initial_conditions: []
+inputs: []
+interventions: []
+name: water_supply
+outputs:
+- name: water_supply_reservoir_end_volumes
+  dims:
+  - water_supply_days_into_year
+  - water_supply_reservoirs
+  dtype: float
+  unit: ML
+- name: water_supply_arc_flows
+  dims:
+  - water_supply_days_into_year
+  - water_supply_arcs
+  dtype: float
+  unit: ML/day
+parameters: []
+path: models/water_supply/run.py

config/sos_models/water_supply_only.yml

@@ -0,0 +1,8 @@
+name: water_supply_only
+description: Future water demand for UK
+sector_models: # Select 1 or more of the sector models
+- water_supply
+scenarios: []    # Select 0 or more of the scenario sets
+narratives: []
+scenario_dependencies: []
+model_dependencies: []

models/water_supply/run.py

@@ -0,0 +1,198 @@
+"""Water supply model
+"""
+import os
+import subprocess
+import sys
+
+import numpy as np
+import pandas as pd
+
+from smif.model.sector_model import SectorModel
+
+
+class WaterWrapper(SectorModel):
+    """Water Model Wrapper
+    """
+
+    def before_model_run(self, data_handle=None):
+        """Implement this method to conduct pre-model run tasks
+
+        Arguments
+        ---------
+        data_handle: smif.data_layer.DataHandle
+            Access parameter values (before any model is run, no dependency
+            input data or state is guaranteed to be available)
+
+        Info
+        -----
+        `self.user_data` allows to pass data from before_model_run to main model
+        """
+        pass
+
+    def simulate(self, data_handle):
+        """Runs the water supply model.
+
+        Arguments
+        ---------
+        data_handle : dict
+            A dictionary containing all parameters and model inputs defined in
+            the smif configuration by name
+
+        """
+
+        now = data_handle.current_timestep
+
+        model_dir = os.path.dirname(os.path.realpath(__file__))
+        exe_dir = os.path.join(model_dir, 'exe')
+        nodal_dir = os.path.join(model_dir, 'nodal')
+
+        # This is the executable itself that requires a sysfile and a nodal file
+        wathnet = os.path.join(exe_dir, 'w5_console.exe')
+        assert(os.path.isfile(wathnet)), "Expected to find water supply WATHNET executable at {}".format(wathnet)
+
+        # This is the national model file, which must be edited to specify which days are simulated
+        sysfile = os.path.join(exe_dir, 'National_Model.wat')
+        assert(os.path.isfile(sysfile)), "Expected to find water supply sysfile at {}".format(sysfile)
+
+        sysfile = self.inject_simulation_days(sysfile, now)
+        assert(os.path.isfile(sysfile)), "Expected to find water supply sysfile at {}".format(sysfile)
+
+        # This is the nodal file which is generated from various static data files
+        nodal_file = self.prepare_nodal(nodal_dir, now)
+        assert(os.path.isfile(nodal_file))        
+
+        subprocess.call([
+            wathnet,
+            '-sysfile={}'.format(sysfile),
+            '-nodalfile={}'.format(nodal_file),
+            '-output=RA',
+        ])
+
+        # Output will be the name of the sysfile (modified_model.wat), without the .wat extension
+        # and with (for instance) '_arcFlow.csv' added.
+        #   e.g. `modified_model_arcFlow.csv`        
+        arc_flows = sysfile.replace('.wat', '_arcFlow.csv')
+        assert(os.path.isfile(arc_flows)), "Expected to find water supply arc flow results at {}".format(arc_flows)
+
+        res_vols = sysfile.replace('.wat', '_reservoirEndVolume.csv')
+        assert (os.path.isfile(res_vols)), "Expected to find water supply reservoir results at {}".format(res_vols)
+
+        arc_flows_df = pd.read_csv(
+            arc_flows,
+            sep=',',
+            skiprows=1,  # Row at top called 'Arc flow'
+            usecols=lambda col: col.lower() not in ['replicate', 'day', 'year'],
+        )
+
+        res_vols_df = pd.read_csv(
+            res_vols,
+            sep=',',
+            skiprows=1,  # Row at top called 'Reservoir end volume'
+            usecols=lambda col: col.lower() not in ['replicate', 'day', 'year'],
+        )
+
+        data_handle.set_results('water_supply_arc_flows', arc_flows_df.values)
+        data_handle.set_results('water_supply_reservoir_end_volumes', res_vols_df.values)
+
+    @staticmethod
+    def prepare_nodal(nodal_dir, year_now):
+        """Generates the nodal file necessary for the Wathnet model run.
+
+        Arguments
+        ---------
+        nodal_dir : str
+            Path to the directory containing the necessary files for preparing the nodal file.
+        
+        year_now: int
+            The year to be simulated
+
+        Returns
+        =======
+        output_file : str
+            The path to the generated nodal file
+        """
+
+        # Check necessary files exist
+        prepare_nodal = os.path.join(nodal_dir, 'prepare_nodal.py')
+        assert (os.path.isfile(prepare_nodal)), "Expected to find prepare_nodal script at {}".format(prepare_nodal)
+
+        flow_file = os.path.join(nodal_dir, 'National_WRSM_NatModel_logNSE_obs_11018_1.txt')
+        assert (os.path.isfile(flow_file)), "Expected to find water supply flows file at {}".format(flow_file)
+
+        demand_file = os.path.join(nodal_dir, '001_daily.csv')
+        assert (os.path.isfile(demand_file)), "Expected to find water supply demand file at {}".format(demand_file)
+
+        catchment_file = os.path.join(nodal_dir, 'CatchmentIndex.csv')
+        assert (os.path.isfile(catchment_file)), "Expected to find water supply catchment file at {}".format(catchment_file)
+
+        borehole_file = os.path.join(nodal_dir, 'borehole_forcing_1974_to_2015.csv')
+        assert (os.path.isfile(borehole_file)), "Expected to find water supply borehole data at {}".format(borehole_file)
+
+        nonpublic_file = os.path.join(nodal_dir, 'cams_mean_daily_returns.csv')
+        assert (os.path.isfile(nonpublic_file)), "Expected to find water supply nonpublic data at {}".format(nonpublic_file)
+
+        missing_data_file = os.path.join(nodal_dir, 'missing_data.csv')
+        assert (os.path.isfile(missing_data_file)), "Expected to find water supply missing data at {}".format(missing_data_file)
+
+        output_file = os.path.join(nodal_dir, 'wathnet.nodal')
+
+        subprocess.call([
+            sys.executable, prepare_nodal,
+            '--FlowFile', flow_file,
+            '--DemandFile', demand_file,
+            '--CatchmentFile', catchment_file,
+            '--BoerholeForcingFile', borehole_file,
+            '--NonpublicFile', nonpublic_file,
+            '--MissingDataFile', missing_data_file,
+            '--OutputFile', output_file,
+            '--Year', str(year_now),
+        ])
+
+        assert(os.path.isfile(output_file)), "Expected to find WATHNET nodal file at {}".format(output_file)
+        return output_file
+
+    @staticmethod
+    def inject_simulation_days(sysfile, year_now):
+        """Injects the current year into the sysfile so that the current year is simulated.
+
+        Arguments
+        ---------
+        sysfile : str
+            Path to the sysfile ('National_Model.wat')
+        
+        year_now: int
+            The year to be simulated
+        """
+
+        modified_sysfile = os.path.join(os.path.dirname(sysfile), 'modified_model.wat')
+
+        # This is the unique line directly before the lines we need to edit
+        # We should hit this exactly once, which we use to test that nothing has gone wrong
+        sentinel_line = '! Run options'
+        sentinel_lines_hit = 0
+
+        # String that we want to edit in place
+        new_string = '   {}\n  {}\n   {}\n  {}\n'.format('1', str(year_now), '365', str(year_now))
+
+        with open(modified_sysfile, 'w') as new_file:
+            with open(sysfile, 'r') as old_file:
+
+                # Write every line from the old file directly into the new file
+                for line in old_file:
+                    new_file.write(line)
+
+                    # If we see the sentinel, skip 4 lines in the old file and
+                    # write the replacement string out instead to the new file
+                    if sentinel_line in line:
+
+                        old_file.readline()
+                        old_file.readline()
+                        old_file.readline()
+                        old_file.readline()
+
+                        sentinel_lines_hit += 1
+                        new_file.write(new_string)
+
+        assert(sentinel_lines_hit == 1)
+
+        return modified_sysfile

provision/config.ini

@@ -30,3 +30,9 @@ model_version=5538973b89e3d04d1d799aedfc112cee988572d7
 remote_data=data/et_module/et_module_v0.2.zip
 local_dir=data/et_module
 model_version=25d25afd5df7e24588ade512db62f4a3ac0edc78
+
+[water-supply]
+model_version=6d8642567967f6168b6dc31c97043083bc644b0b
+remote_data=data/water_supply/data_to_prepare_nodal_v3.zip
+local_dir=models/water_supply
+dim_dir=data/dimensions/water_supply

provision/get_data_water_supply.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Expect NISMOD dir as first argument
+base_path=$1
+
+# Read remote_data, local_dir from config.ini
+source <(grep = <(grep -A4 "\[water-supply\]" $base_path/provision/config.ini))
+
+# Ensure the directory exists
+nodal_dir=$local_dir/nodal
+mkdir -p $nodal_dir
+
+# Download data
+python $base_path/provision/get_data.py $remote_data $nodal_dir

provision/install_water_supply.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Expect NISMOD dir as first argument
+base_path=$1
+
+# Read remote_data, local_dir from config.ini
+source <(grep = <(grep -A4 "\[water-supply\]" $base_path/provision/config.ini))
+
+# Locations for the git repo (temporary) and the nodal-related files
+repo_dir=$local_dir/repo
+nodal_dir=$local_dir/nodal
+exe_dir=$local_dir/exe
+dim_dir=$dim_dir
+
+# Clone repo and copy necessary files to the model directory
+mkdir -p $repo_dir
+git clone [email protected]:nismod/water_supply.git $repo_dir || exit "$?"
+pushd $repo_dir
+    git checkout $model_version
+popd
+
+mkdir -p $exe_dir
+cp $repo_dir/wathnet/w5_console.exe $exe_dir
+cp $repo_dir/models/National_Model.wat $exe_dir
+
+# Seems to be necessary to add execution to the wathnet exe
+chmod +x $exe_dir/w5_console.exe
+
+# Copy files needed for creating nodal file. This will be superseded by data being fed directly.
+mkdir -p $nodal_dir
+cp $local_dir/repo/scripts/preprocessing/prepare_nodal.py $nodal_dir
+
+# Copy data dimensions
+mkdir -p $dim_dir
+cp $repo_dir/data_dimensions/* $dim_dir
+
+# Clean up the cloned repo
+rm -rf $repo_dir

provision/vm_provision.sh

@@ -213,6 +213,9 @@ bash -x $base_path/provision/install_transport.sh $base_path
 bash -x $base_path/provision/get_data_et_module.sh $base_path
 bash -x $base_path/provision/install_et_module.sh $base_path
 
+# Water supply
+bash -x $base_path/provision/install_water_supply.sh $base_path
+
 #
 # User config
 #