From 95e145f84aa3378e7735997c06637d9c3c89441a Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 1 Sep 2020 17:33:19 -0600 Subject: [PATCH 001/145] Handle list-like input to InternalCoupling.write_BCs(...,fieldname=...) Fieldname may represent a - scalar BC: name of column in dataframe (or 0 for all zeroes) - vector BC: list of 3 columns (names and/or 0's) Example: ``` to_sowfa = InternalCoupling(outdir,df_hist,dateref=simstart) to_sowfa.write_BCs('surfaceTemperatureFluxTable', fieldname='hfx') to_sowfa.write_BCs('qwallTable', fieldname=[0,0,'hfx']) ``` for which 'hfx' is a column in the time-history dataframe `df_hist` --- mmctools/coupling/sowfa.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/mmctools/coupling/sowfa.py b/mmctools/coupling/sowfa.py index b48f584..e530553 100755 --- a/mmctools/coupling/sowfa.py +++ b/mmctools/coupling/sowfa.py @@ -124,19 +124,35 @@ def write_BCs(self, # extract time and height array ts = self.df.t_index.values nt = ts.size - - # assert field exists and is complete - assert(fieldname in self.df.columns), 'Field '+fieldname+' not in df' - assert(~pd.isna(self.df[fieldname]).any()), 'Field '+fieldname+' is not complete (contains NaNs)' - - # scale field with factor, - # e.g., scale heat flux with fact=-1 to follow OpenFOAM sign convention - fieldvalues = fact * self.df[fieldname].values - - with open(os.path.join(self.dpath,fname),'w') as fid: + + # check if scalar or vector + if isinstance(fieldname, (list,tuple)): + assert len(fieldname) == 3, 'expected 3 vector components' + fieldnames = fieldname + fmt = [' (%g', '(%.12g', '%.12g', '%.12g))',] + else: + fieldnames = [fieldname] fmt = [' (%g', '%.12g)',] + + # setup output data, assert field(s) exists and is complete + fieldvalues = [] + for fieldname in fieldnames: + if fieldname == 0: + fieldvalues.append(np.zeros_like(ts)) + else: + assert(fieldname in self.df.columns), 'Field '+fieldname+' not in df' + assert(~pd.isna(self.df[fieldname]).any()), 'Field '+fieldname+' is not complete (contains NaNs)' + fieldvalues.append(self.df[fieldname].values) + + # scale field with factor, + # e.g., scale heat flux with fact=-1 to follow OpenFOAM sign convention + fieldvalues[-1] = fact * fieldvalues[-1] + + fieldvalues = np.array(fieldvalues).T # result: fieldvalues.shape==(nt, ndim) + + with open(os.path.join(self.dpath,fname),'w') as fid: np.savetxt(fid,np.concatenate((ts.reshape((nt,1)), - fieldvalues.reshape((nt,1)) + fieldvalues ),axis=1),fmt=fmt) return From a873ffcf1a16c2fba0ef0922a72d38667c0c183e Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 1 Sep 2020 17:40:32 -0600 Subject: [PATCH 002/145] Update docstring, cleanup comments --- mmctools/coupling/sowfa.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mmctools/coupling/sowfa.py b/mmctools/coupling/sowfa.py index e530553..87b0bba 100755 --- a/mmctools/coupling/sowfa.py +++ b/mmctools/coupling/sowfa.py @@ -113,15 +113,17 @@ def write_BCs(self, ===== fname : str Filename - fieldname : str - Name of the field to be written out + fieldname : str or list-like + Name of the scalar field (or a list of names of vector field + components) to be written out; 0 may be substituted to + indicate an array of zeroes fact : float Scale factor for the field, e.g., to scale heat flux to follow OpenFOAM sign convention that boundary fluxes are positive if directed outward """ - # extract time and height array + # extract time array ts = self.df.t_index.values nt = ts.size @@ -134,14 +136,16 @@ def write_BCs(self, fieldnames = [fieldname] fmt = [' (%g', '%.12g)',] - # setup output data, assert field(s) exists and is complete + # assert field(s) exists and is complete, setup output data fieldvalues = [] for fieldname in fieldnames: if fieldname == 0: fieldvalues.append(np.zeros_like(ts)) else: - assert(fieldname in self.df.columns), 'Field '+fieldname+' not in df' - assert(~pd.isna(self.df[fieldname]).any()), 'Field '+fieldname+' is not complete (contains NaNs)' + assert(fieldname in self.df.columns), \ + 'Field '+fieldname+' not in df' + assert(~pd.isna(self.df[fieldname]).any()), \ + 'Field '+fieldname+' is not complete (contains NaNs)' fieldvalues.append(self.df[fieldname].values) # scale field with factor, @@ -188,7 +192,7 @@ def write_ICs(self, if not field in df.columns: df.loc[:,field] = 0.0 - # extract time and height array + # extract height array zs = df.height.values nz = zs.size From e37be01ecdf1a486bb64d8ba45150477de7397ff Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 2 Sep 2020 14:52:42 -0600 Subject: [PATCH 003/145] Work with copies of input variables so that inputs (e.g., series from a dataframe) do not get inadvertently modified --- mmctools/helper_functions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 1bb9a42..ab78820 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -15,10 +15,11 @@ def e_s(T, celsius=False, model='Tetens'): """Calculate the saturation vapor pressure of water, $e_s$ [mb] given the air temperature ([K] by default). """ + T = T.copy() if celsius: # input is deg C T_degC = T - T = T + 273.15 + T = T_degC + 273.15 else: # input is in Kelvin T_degC = T - 273.15 @@ -82,9 +83,10 @@ def T_to_Tv(T,p=None,RH=None,e=None,w=None,Td=None, pressures of water vapor and dry air (e, pd [mbar]); or dewpoint temperature (Td). """ + T = T.copy() if celsius: T_degC = T - T += 273.15 + T = T_degC + 273.15 else: T_degC = T - 273.15 if (p is not None) and (RH is not None): @@ -126,7 +128,7 @@ def T_to_Tv(T,p=None,RH=None,e=None,w=None,Td=None, elif (Td is not None) and (p is not None): # From National Weather Service, using Tetens' formula: # https://www.weather.gov/media/epz/wxcalc/vaporPressure.pdf - Td_degC = Td + Td_degC = Td.copy() if not celsius: Td_degC -= 273.15 e = e_s(Td_degC, celsius=True, model='Tetens') From 6b9ec31d2d99e53bce0a6ae62de85547388f6e7f Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 2 Sep 2020 14:55:29 -0600 Subject: [PATCH 004/145] Fix warnings about comparisons with is rather than == --- mmctools/helper_functions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index ab78820..81b214c 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -558,7 +558,7 @@ def model4D_spectra(ds,spectra_dim,average_dim,vert_levels,horizontal_locs,fld,f f, Pxxfc = welch(series, fs, window=win, noverlap=overlap, nfft=nblock, return_onesided=False, detrend='constant') Pxxf = np.multiply(np.real(Pxxfc),np.conj(Pxxfc)) - if it is 0: + if it == 0: Puuf_cum[cnt_lvl,cnt_i,:] = Pxxf else: Puuf_cum[cnt_lvl,cnt_i,:] = Puuf_cum[cnt_lvl,cnt_i,:] + Pxxf @@ -613,7 +613,7 @@ def model4D_spatial_spectra(ds,spectra_dim,vert_levels,horizontal_locs,fld,fldMe f, Pxxfc = welch(series, fs, window=win, noverlap=overlap, nfft=nblock, return_onesided=False, detrend='constant') Pxxf = np.multiply(np.real(Pxxfc),np.conj(Pxxfc)) - if it is 0: + if it == 0: Puuf_cum[cnt_lvl,cnt_i,:] = Pxxf else: Puuf_cum[cnt_lvl,cnt_i,:] = Puuf_cum[cnt_lvl,cnt_i,:] + Pxxf @@ -690,7 +690,7 @@ def model4D_cospectra(ds,spectra_dim,average_dim,vert_levels,horizontal_locs,fld nfft=nblock, return_onesided=False, detrend='constant') Pxxf = (np.multiply(np.real(Pxxfc0),np.conj(Pxxfc1))+ np.multiply(np.real(Pxxfc1),np.conj(Pxxfc0))) - if it is 0: + if it == 0: Puuf_cum[cnt_lvl,cnt_i,:] = Pxxf else: Puuf_cum[cnt_lvl,cnt_i,:] = Puuf_cum[cnt_lvl,cnt_i,:] + Pxxf @@ -751,7 +751,7 @@ def model4D_spatial_cospectra(ds,spectra_dim,vert_levels,horizontal_locs,fldv0,f f, Pxxfc1 = welch(series1, fs, window=win, noverlap=overlap, nfft=nblock, return_onesided=False, detrend='constant') Pxxf = (np.multiply(np.real(Pxxfc0),np.conj(Pxxfc1))+np.multiply(np.real(Pxxfc1),np.conj(Pxxfc0))) - if it is 0: + if it == 0: Puuf_cum[cnt_lvl,cnt_i,:] = Pxxf else: Puuf_cum[cnt_lvl,cnt_i,:] = Puuf_cum[cnt_lvl,cnt_i,:] + Pxxf @@ -873,7 +873,7 @@ def model4D_spatial_pdfs(ds,pdf_dim,vert_levels,horizontal_locs,fld,fldMean,bins y = (ds[fld].isel(datetime=it,nz=level,nx=iLoc)-ds[fldMean].isel(datetime=it,nz=level,nx=iLoc)) #y = np.ndarray.flatten(dist.isel(nz=level,nx=iLoc).values) hist,bin_edges=np.histogram(y, bins=bins_vector) - if it is 0: + if it == 0: hist_cum[cnt_lvl,cnt_i,:] = hist else: hist_cum[cnt_lvl,cnt_i,:] = hist_cum[cnt_lvl,cnt_i,:] + hist From 62d91079ec6a0d0c47ff3d56e270d452aaf4266c Mon Sep 17 00:00:00 2001 From: ewquon Date: Thu, 3 Sep 2020 13:03:05 -0600 Subject: [PATCH 005/145] Allow use of mmctools.wrf without wrf-python module --- mmctools/wrf/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 35a08d4..3a237a5 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -23,7 +23,6 @@ from scipy.spatial import KDTree from scipy.interpolate import interp1d, LinearNDInterpolator import netCDF4 -import wrf as wrfpy from ..helper_functions import calc_wind @@ -1163,6 +1162,7 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, (i.e., want a range of non-interpolated heights), and you only care about data that are below a certain vertical index. """ + import wrf as wrfpy TH0 = 300.0 #WRF convention base-state theta = 300.0 K dims_dict = { 'Time':'datetime', From 9aa8beb2fc8d0ba9599c9fc197ee60b3b6c1fb5b Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 13 Jan 2021 23:25:12 -0700 Subject: [PATCH 006/145] Add **kwargs to tsout_seriesReader() This is passed to combine_towers() and, subsequently, to_dataframe() and handles extra options for tsout processing --- mmctools/wrf/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 3a237a5..618e102 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1084,7 +1084,8 @@ def combine_towers(fdir, restarts, simulation_start, fname, def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest, structure='ordered', time_step=None, - heights=None, height_var='heights', select_tower=None): + heights=None, height_var='heights', select_tower=None, + **kwargs): ''' This will combine a series of tslist output over time and location based on the path to the case (fdir), the restart directories (restarts), a model start time @@ -1130,7 +1131,8 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest tower_names = good_towers dsF = combine_towers(fdir,restarts,simulation_start_time,tower_names, structure=structure, time_step=time_step, - heights=heights, height_var=height_var) + heights=heights, height_var=height_var, + **kwargs) return dsF From 4b4a4eb2accc37e48914a601aa97940b48b0e32a Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 13 Jan 2021 23:26:28 -0700 Subject: [PATCH 007/145] Fix rounding bug with default wrf.Tower.to_dataframe() For large mesoscale timesteps >= 1 s, previous behavior (without a specified time-step size, default) would round timestamps to the nearest second and deal with any single-to-double precision conversion issues-- providing a clean dataframe index as a result. However, for smaller timesteps that are fractions of a second, this results in dataframes with non-unique indices. The new rounding is to the nearest millisecond. With 6 decimal places in the tsout towers, the actual output precision is 1e-6 h or 0.0036 s ~ O(1e-3) s --- mmctools/wrf/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 618e102..c3eb7ed 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -481,7 +481,7 @@ def to_dataframe(self,start_time, start_time = pd.to_datetime(start_time) if time_step is None: times = start_time + pd.to_timedelta(self.time, unit=time_unit) - times = times.round(freq='1s') + times = times.round(freq='1ms') times.name = 'datetime' else: timestep = pd.to_timedelta(time_step, unit='s') From 519f2139798a1d0b666bcd0f3960b403b6f7e56b Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 20 Jan 2021 14:05:11 -0700 Subject: [PATCH 008/145] Fix momentum flux calculation in model4D_calcQOIs() --- mmctools/helper_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 81b214c..1778d8d 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -492,7 +492,7 @@ def model4D_calcQOIs(ds,mean_dim,data_type='wrfout', mean_opt='static', lowess_d ds['vw'] = ds_perts['v']*ds_perts['w'] ds['wth'] = ds_perts['w']*ds_perts['theta'] ds['UU'] = ds_perts['wspd']**2 - ds['Uw'] = ds_perts['wspd']**2 + ds['Uw'] = ds_perts['wspd']*ds_perts['w'] ds['TKE'] = 0.5*np.sqrt(ds['UU']+ds['ww']) ds.attrs['MEAN_OPT'] = mean_opt if mean_opt == 'lowess': From 45e091b4f2e82567975a8365e6b581ac5987d4ed Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 23 Jan 2021 15:40:02 -0700 Subject: [PATCH 009/145] Handle single-prec issue when calling Tower.to_dataframe() with a specified time_step --- mmctools/wrf/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index c3eb7ed..8fed1c2 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -484,12 +484,15 @@ def to_dataframe(self,start_time, times = times.round(freq='1ms') times.name = 'datetime' else: - timestep = pd.to_timedelta(time_step, unit='s') - endtime = start_time + self.nt*timestep + pd.to_timedelta(np.round(self.time[0],decimals=1),unit='h') - times = pd.date_range(start=start_time+timestep+pd.to_timedelta(np.round(self.time[0],decimals=1),unit='h'), - end=endtime, + timedelta = pd.to_timedelta(time_step, unit='s') + # note: time is in hours and _single-precision_ + Nsteps0 = np.round(self.time[0] / (time_step/3600)) + toffset0 = Nsteps0 * timedelta + times = pd.date_range(start=start_time+toffset0, + freq=timedelta, periods=self.nt, name='datetime') + times = times.round(freq='1ms') # combine (and interpolate) time-height data # - note 1: self.[varn].shape == self.height.shape == (self.nt, self.nz) # - note 2: arraydata.shape == (self.nt, len(varns)*self.nz) From 9c1b66c2bea213a0c0621c9b3beec7608f24f7d4 Mon Sep 17 00:00:00 2001 From: ewquon Date: Tue, 2 Feb 2021 09:51:42 -0700 Subject: [PATCH 010/145] Update USGS map url --- mmctools/coupling/terrain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 19dba19..1c1e756 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -361,7 +361,7 @@ def _get_bounds_from_metadata(self): def download(self): """This is just a stub""" print('Data must be manually downloaded!') - print('Go to https://viewer.nationalmap.gov/basic/,') + print('Go to https://apps.nationalmap.gov/downloader/#/,') print('select "Data > Elevation Products (3DEP)"') print('and then click "Find Products"') From 6c2727fd98fc535b1f4a907620cb010fa68ba1a6 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Feb 2021 10:18:22 -0700 Subject: [PATCH 011/145] Catch CalledProcessError --- mmctools/coupling/terrain.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 1c1e756..55020f2 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -12,7 +12,7 @@ - install with `conda install -c conda-forge rasterio` or `pip install rasterio` - note: like the elevation package, this also depends on gdal """ -import os,glob +import sys,os,glob import numpy as np from scipy.interpolate import RectBivariateSpline @@ -287,7 +287,14 @@ def download(self,cleanup=True): if not os.path.isdir(dpath): print('Creating path',dpath) os.makedirs(dpath) - elevation.clip(self.bounds, product=self.product, output=self.tiffdata) + try: + elevation.clip(self.bounds, product=self.product, output=self.tiffdata) + except: + info = sys.exc_info() + print(info[0]) + print(info[1]) + print('') + print('Note: Have elevation and gdal been installed properly?') if cleanup: elevation.clean() From 0e1ca5ddf2509113eaa9b4c4aea3cc19f99e5390 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Feb 2021 10:36:28 -0700 Subject: [PATCH 012/145] Handle tif paths with spaces elevations.clip() doesn't properly handle file paths with spaces when calling gdal_translate --- mmctools/coupling/terrain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 55020f2..a0e0de6 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -287,8 +287,9 @@ def download(self,cleanup=True): if not os.path.isdir(dpath): print('Creating path',dpath) os.makedirs(dpath) + escapedpath = self.tiffdata.replace('\ ',' ').replace(' ','\ ') try: - elevation.clip(self.bounds, product=self.product, output=self.tiffdata) + elevation.clip(self.bounds, product=self.product, output=escapedpath) except: info = sys.exc_info() print(info[0]) From dd8a68f7576db12505889cf38842e816a2cc0ea8 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Feb 2021 19:24:36 -0700 Subject: [PATCH 013/145] Handle new ISO XML metadata format Legacy format (with root tag "metadata") has been retained. XML namespace definitions have been hardcoded for geospatial/geographic metadata (GMD/GCO, respectively) because there does not appear to be a straightforward way to extract the xmlns attributes from the root tag using ElementTree. --- mmctools/coupling/terrain.py | 70 ++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index a0e0de6..2bac7fd 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -21,6 +21,13 @@ from rasterio import transform, warp from rasterio.crs import CRS +# hard-coded here because ElementTree doesn't appear to have any +# straightforward way to access the xmlns root attributes +ISO_namespace = { + 'gmd': 'http://www.isotc211.org/2005/gmd', + 'gco': 'http://www.isotc211.org/2005/gco', +} + class Terrain(object): @@ -110,6 +117,8 @@ def to_terrain(self,dx,dy=None,resampling=warp.Resampling.bilinear): SE_x,SE_y = self.to_xy(south,east) Lx = SE_x - orix Ly = oriy - SE_y + print(Lx,Ly,dx,dy) + print(west,south,east,north) Nx = int(Lx / dx) Ny = int(Ly / dy) @@ -337,7 +346,7 @@ def __init__(self,latlon_bounds=None,fpath='terrain.tif'): fpath : str Location of downloaded GeoTIFF (*.tif) data. """ - self.metadata = self._read_metadata(fpath) + self._read_metadata(fpath) if latlon_bounds is None: latlon_bounds = self._get_bounds_from_metadata() print('Bounds:',latlon_bounds) @@ -350,20 +359,61 @@ def _read_metadata(self,fpath): metadata = ElementTree.parse(xmlfile).getroot() except IOError: self.have_metadata = False - metadata = None else: - assert metadata.tag == 'metadata' - print('Source CRS datum:',metadata.find('./spref/horizsys/geodetic/horizdn').text) + if not metadata.tag.endswith('MD_Metadata'): + assert metadata.tag in ['metadata','gmd:MD_Metadata','modsCollection'] + if metadata.tag == 'metadata': + # legacy metadata + print('Source CRS datum:',metadata.find('./spref/horizsys/geodetic/horizdn').text) + elif metadata.tag == 'modsCollection': + # MODS XML + print(metadata.find('./mods/titleInfo/title').text) + raise NotImplementedError('MODS XML detected -- use ISO XML instead') + else: + # ISO XML + title = metadata.find( + '/'.join([ + 'gmd:identificationInfo', + 'gmd:MD_DataIdentification', + 'gmd:citation', + 'gmd:CI_Citation', + 'gmd:title', + 'gco:CharacterString', + ]), + ISO_namespace + ).text + print(title) self.have_metadata = True - return metadata + self.metadata = metadata def _get_bounds_from_metadata(self): assert self.have_metadata - bounding = self.metadata.find('./idinfo/spdom/bounding') - bounds = [ - float(bounding.find(bcdir+'bc').text) - for bcdir in ['west','south','east','north'] - ] + if self.metadata.tag == 'metadata': + # legacy metadata + bounding = self.metadata.find('./idinfo/spdom/bounding') + bounds = [ + float(bounding.find(bcdir+'bc').text) + for bcdir in ['west','south','east','north'] + ] + else: + # ISO XML + extent = self.metadata.find( + 'gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent', + ISO_namespace + ) + bbox = extent.find( + 'gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox', + ISO_namespace + ) + bounds = [ + float(bbox.find(f'gmd:{bound}/gco:Decimal',ISO_namespace).text) + for bound in [ + 'westBoundLongitude', + 'southBoundLatitude', + 'eastBoundLongitude', + 'northBoundLatitude', + ] + ] return bounds def download(self): From 5104331b3997a9cd060ceb4fda5c8e2a95bb4a45 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Feb 2021 20:41:32 -0700 Subject: [PATCH 014/145] Add slope calculation function --- mmctools/coupling/terrain.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 2bac7fd..ec6b03b 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -253,6 +253,7 @@ def ytransect(self,xy=None,latlon=None,wdir=180.0,yrange=(None,None)): return y-refloc[1], z + class SRTM(Terrain): """Class for working with Shuttle Radar Topography Mission (SRTM) data""" data_products = { @@ -475,3 +476,36 @@ def combine_raster_data(filelist,dtype=Terrain,latlon_bounds=None, bounds_max = bounds.max(axis=0) return [bounds_min[0],bounds_min[1],bounds_max[2],bounds_max[3]] +def calc_slope(x,y,z): + """Calculate local terrain slope based on project grid + + Notes: + - Uses neighborhood method (weighted second-order difference, based on + 3x3 stencil) + - Slopes are not calculated at edge points (i.e., locations where a 3x3 + stencil cannot be formed) + + Usage + ===== + x,y,z : numpy array + Equally sized 2-D arrays; if not specified, then the full terrain + will be used + """ + dx = x[1,0] - x[0,0] + dy = y[0,1] - y[0,0] + slope = np.empty_like(z) + slope[:,:] = np.nan + z1 = z[ :-2, 2: ] # upper left + z2 = z[ 1:-1, 2: ] # upper middle + z3 = z[ 2: , 2: ] # upper right + z4 = z[ :-2, 1:-1] # center left + #z5 = z[ 1:-1, 1:-1] # center + z6 = z[ 2: , 1:-1] # center right + z7 = z[ :-2, :-2] # lower left + z8 = z[ 1:-1, :-2] # lower middle + z9 = z[ 2: , :-2] # lower right + dz_dx = ((z3 + 2*z6 + z9) - (z1 + 2*z4 + z7)) / (8*dx) + dz_dy = ((z1 + 2*z2 + z3) - (z7 + 2*z8 + z9)) / (8*dy) + rise_run = np.sqrt(dz_dx**2 + dz_dy**2) + slope[1:-1,1:-1] = np.degrees(np.arctan(rise_run)) + return slope From 6966ecbcd8b65a10a3d08ce5ea8d8dd220d30ed6 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Feb 2021 20:48:17 -0700 Subject: [PATCH 015/145] Minor cleanup --- mmctools/coupling/terrain.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index ec6b03b..4d93d1b 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -117,8 +117,6 @@ def to_terrain(self,dx,dy=None,resampling=warp.Resampling.bilinear): SE_x,SE_y = self.to_xy(south,east) Lx = SE_x - orix Ly = oriy - SE_y - print(Lx,Ly,dx,dy) - print(west,south,east,north) Nx = int(Lx / dx) Ny = int(Ly / dy) @@ -253,7 +251,6 @@ def ytransect(self,xy=None,latlon=None,wdir=180.0,yrange=(None,None)): return y-refloc[1], z - class SRTM(Terrain): """Class for working with Shuttle Radar Topography Mission (SRTM) data""" data_products = { From 237f7285d863b282cb4c59c784f028b506f80c7b Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 12 Feb 2021 13:24:43 -0700 Subject: [PATCH 016/145] Allow latest versions of packages --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 063f494..beed2df 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,10 @@ REQUIRED = [ # core 'matplotlib>=3', - 'numpy==1.18.1', - 'scipy==1.4.1', - 'pandas==1.0.1', - 'xarray==0.15.0', + 'numpy>=1.18.1', + 'scipy>=1.4.1', + 'pandas>=1.0.1', + 'xarray>=0.15.0', 'netcdf4>=1.5.1', 'dask>=2.10.1', 'utm>=0.5.0', From 11663c1ab7cc9f055b4eac6e3f24e6a22e522217 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 13 Feb 2021 16:05:12 -0700 Subject: [PATCH 017/145] Add extrapolate option for plot_timehistory_at_height To disable extrapolation for heights outside of the data range --- mmctools/plotting.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mmctools/plotting.py b/mmctools/plotting.py index 6772972..1abbc34 100644 --- a/mmctools/plotting.py +++ b/mmctools/plotting.py @@ -337,6 +337,7 @@ def plot_timeheight(datasets, def plot_timehistory_at_height(datasets, fields=None, heights=None, + extrapolate=True, fig=None,ax=None, fieldlimits=None, timelimits=None, @@ -377,6 +378,9 @@ def plot_timehistory_at_height(datasets, value. 'all' means the time history for all heights in the datasets will be plotted (in this case all datasets should have the same heights) + extrapolate : bool + If false, then output height(s) outside the data range will + not be plotted; default is true for backwards compatibility fig : figure handle Custom figure handle. Should be specified together with ax ax : axes handle, or list or numpy ndarray with axes handles @@ -510,6 +514,7 @@ def plot_timehistory_at_height(datasets, if (not heightvalues is None) and (not all([h in heightvalues for h in args.heights])): df_pivot = _get_pivot_table(df,'height',available_fields) pivoted = True + fill_value = 'extrapolate' if extrapolate else np.nan if debug: print('Pivoting '+dfname) else: pivoted = False @@ -524,6 +529,15 @@ def plot_timehistory_at_height(datasets, continue for k, height in enumerate(args.heights): + # Check if height is outside of data range + if (height > np.max(heightvalues)) or (height < np.min(heightvalues)): + if extrapolate: + if debug: + print('Extrapolating field "'+field+'" at z='+str(height)+' in dataset '+dfname) + else: + print('Warning: field "'+field+'" not available at z='+str(height)+' in dataset '+dfname) + continue + # Store plotting options in dictionary # Set default linestyle to '-' and no markers plotting_properties = { @@ -567,7 +581,7 @@ def plot_timehistory_at_height(datasets, # Extract data from dataframe if pivoted: - signal = interp1d(heightvalues,_get_pivoted_field(df_pivot,field).values,axis=-1,fill_value="extrapolate")(height) + signal = interp1d(heightvalues,_get_pivoted_field(df_pivot,field).values,axis=-1,fill_value=fill_value)(height) else: slice_z = _get_slice(df,height,'height') signal = _get_field(slice_z,field).values From d71a65206b39880f6f1f957ddd052c31ba87117b Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 13 Feb 2021 17:41:08 -0700 Subject: [PATCH 018/145] Fix extrapolation check for timeseries --- mmctools/plotting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmctools/plotting.py b/mmctools/plotting.py index 1abbc34..7caee31 100644 --- a/mmctools/plotting.py +++ b/mmctools/plotting.py @@ -530,7 +530,8 @@ def plot_timehistory_at_height(datasets, for k, height in enumerate(args.heights): # Check if height is outside of data range - if (height > np.max(heightvalues)) or (height < np.min(heightvalues)): + if (heightvalues is not None) and \ + ((height > np.max(heightvalues)) or (height < np.min(heightvalues))): if extrapolate: if debug: print('Extrapolating field "'+field+'" at z='+str(height)+' in dataset '+dfname) From 7da0952389b04dab75600ab6d65308ad52903eea Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 23 Feb 2021 07:55:41 -0700 Subject: [PATCH 019/145] Adding MERRA2 as downloadable option. --- mmctools/wrf/preprocessing.py | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 8886398..b4d967f 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -178,7 +178,47 @@ def download(self,datetimes,path=None): datetimes=datetimes, fields=['regn128sc']) + +class MERRA2(RDADataset): + """MERRA2 Global Atmosphere Forcing Data + + Description:https://rda.ucar.edu/datasets/ds313.3/ + """ + + def set_resolution(self,resolution_deg=0): + if resolution_deg == 2: + self.res_str = '1.9x2.5' + elif resolution_deg == 1: + self.res_str = '0.9x1.25' + elif resolution_deg == 0.5: + self.res_str = '0.5x0.63' + else: + self.res_str = 'orig_res' + + def download(self,datetimes,path=None): + """Download data at specified datetimes. + + Files to download: + - https://rda.ucar.edu/data/ds313.3/0.9x1.25/YYYY/MERRA2_0.9x1.25_YYYYMMDD.nc + Usage + ===== + datetimes : timestamp or list of timestamps + Datetime, e.g., output from + pd.date_range(startdate,enddate,freq='21600s') + path : str, optional + Path to directory in which to save grib files + """ + if path is None: + path = '.' + else: + os.makedirs(path,exist_ok=True) + super().download(urlpath='ds313.3/{}/%Y/MERRA2{}_%Y%m%d.nc'.format(self.res_str), + path=path, + datetimes=datetimes) + + + class CDSDataset(object): """Class to help with downloading initial and boundary conditions from the Copernicus Climate Data Store (CDS) to use with WPS. From fa149407574a82e5f87d66fda1bad340af4e8a85 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 23 Feb 2021 07:58:55 -0700 Subject: [PATCH 020/145] Need to delete these assertions. --- mmctools/wrf/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 3a237a5..4dc9e5b 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -368,15 +368,15 @@ def _create_datadict(self,varns,unstagger=False,staggered_vars=['ph']): datadict[varn] = ((tsdata[:,1:] + tsdata[:,:-1]) / 2).ravel() elif varn == 'th': # theta is a special case - assert np.all(tsdata[:,-1] == 300), 'Unexpected nonzero value for theta' + #assert np.all(tsdata[:,-1] == 300), 'Unexpected nonzero value for theta' # drop the trailing 0 for already unstaggered quantities datadict[varn] = tsdata[:,:-1].ravel() else: # other quantities already unstaggered - if not varn == 'ww': - # don't throw a warning if w is already unstaggered by the code - # last value is (w(model top) + 0.0)/2.0 - assert np.all(tsdata[:,-1] == 0), 'Unexpected nonzero value for '+varn + #if not varn == 'ww': + # # don't throw a warning if w is already unstaggered by the code + # # last value is (w(model top) + 0.0)/2.0 + # assert np.all(tsdata[:,-1] == 0), 'Unexpected nonzero value for '+varn # drop the trailing 0 for already unstaggered quantities datadict[varn] = tsdata[:,:-1].ravel() else: From e299bfa9a4f07f3600cd349e0f0842e8dde33db2 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 23 Feb 2021 08:01:55 -0700 Subject: [PATCH 021/145] Add new sfc vars and remove assertions --- mmctools/wrf/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 35a08d4..077d987 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -57,6 +57,8 @@ 'RAINC', # rainfall from a cumulus scheme [mm] 'RAINNC', # rainfall from an explicit scheme [mm] 'CLW', # total column-integrated water vapor and cloud variables + 'QFX', # Vapor flux (upward is positive) [g m^-2 s^-1] + 'UST', # u* from M-O ] def _get_dim(wrfdata,dimname): @@ -369,15 +371,15 @@ def _create_datadict(self,varns,unstagger=False,staggered_vars=['ph']): datadict[varn] = ((tsdata[:,1:] + tsdata[:,:-1]) / 2).ravel() elif varn == 'th': # theta is a special case - assert np.all(tsdata[:,-1] == 300), 'Unexpected nonzero value for theta' + #assert np.all(tsdata[:,-1] == 300), 'Unexpected nonzero value for theta' # drop the trailing 0 for already unstaggered quantities datadict[varn] = tsdata[:,:-1].ravel() else: # other quantities already unstaggered - if not varn == 'ww': + #if not varn == 'ww': # don't throw a warning if w is already unstaggered by the code # last value is (w(model top) + 0.0)/2.0 - assert np.all(tsdata[:,-1] == 0), 'Unexpected nonzero value for '+varn + # assert np.all(tsdata[:,-1] == 0), 'Unexpected nonzero value for '+varn # drop the trailing 0 for already unstaggered quantities datadict[varn] = tsdata[:,:-1].ravel() else: From 676fb2c60a6511276ef02ec648ddbdcf2fef62b5 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 24 Feb 2021 12:11:01 -0700 Subject: [PATCH 022/145] Bug fix: improper calculation of spectra There is no consistency in these model4D_spectra functions... Implementing the use of an existing function, power_spectral_density to be used instead of this so there is consistency. --- mmctools/helper_functions.py | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 81b214c..741908d 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -293,7 +293,10 @@ def power_spectral_density(df,tstart=None,interval=None,window_size='10min', # Determine sampling rate and samples per window dts = np.diff(timevalues.unique())/timescale dt = dts[0] - nperseg = int( pd.to_timedelta(window_size)/pd.to_timedelta(dt,'s') ) + if type(window_type) is str: + nperseg = int( pd.to_timedelta(window_size)/pd.to_timedelta(dt,'s') ) + else: + nperseg = len(window_type) assert(np.allclose(dts,dt)),\ 'Timestamps must be spaced equidistantly' @@ -540,31 +543,47 @@ def model4D_spectra(ds,spectra_dim,average_dim,vert_levels,horizontal_locs,fld,f fs = 1 / dt overlap = 0 win = hamming(nblock, True) #Assumed non-periodic in the spectra_dim - Puuf_cum = np.zeros((len(vert_levels),len(horizontal_locs),ds.dims[spectra_dim])) + + init_Puuf_cum = True for cnt_lvl,level in enumerate(vert_levels): # loop over levels print('grabbing a slice...') spec_start = time.time() series_lvl = ds[fld].isel(nz=level)-ds[fldMean].isel(nz=level) + series_lvl.name = 'varn' print(time.time() - spec_start) for cnt_i,iLoc in enumerate(horizontal_locs): # loop over x for cnt,it in enumerate(range(ds.dims[average_dim])): # loop over y if spectra_dim == 'datetime': series = series_lvl.isel(nx=iLoc,ny=it) + if (type(series) == xr.Dataset) or (type(series) == xr.DataArray): + series = series.to_dataframe() + for key in series.keys(): + if key != 'varn': + series = series.drop([key],axis=1) + elif 'y' in spectra_dim: series = series_lvl.isel(nx=iLoc,datetime=it) else: print('Please choose spectral_dim of \'ny\', or \'datetime\'') - f, Pxxfc = welch(series, fs, window=win, noverlap=overlap, - nfft=nblock, return_onesided=False, detrend='constant') - Pxxf = np.multiply(np.real(Pxxfc),np.conj(Pxxfc)) + #f, Pxxfc = welch(series, fs, window=win, noverlap=overlap, + # nfft=nblock, return_onesided=False, detrend='constant') + #Pxxf = np.multiply(np.real(Pxxfc),np.conj(Pxxfc)) + + Pxxf = power_spectral_density(series,window_type=win,detrend='constant') if it == 0: - Puuf_cum[cnt_lvl,cnt_i,:] = Pxxf + if init_Puuf_cum: + Puuf_cum = np.zeros((len(vert_levels),len(horizontal_locs),len(Pxxf))) + init_Puuf_cum = False + Puuf_cum[cnt_lvl,cnt_i,:] = Pxxf.varn + sum_count = 1 else: - Puuf_cum[cnt_lvl,cnt_i,:] = Puuf_cum[cnt_lvl,cnt_i,:] + Pxxf - Puuf = 2.0*(1.0/cnt)*Puuf_cum[:,:,:(np.floor(ds.dims[spectra_dim]/2).astype(int))] ###2.0 is to account for the dropping of the negative side of the FFT - f = f[:(np.floor(ds.dims[spectra_dim]/2).astype(int))] - + Puuf_cum[cnt_lvl,cnt_i,:] += Pxxf.varn + sum_count += 1 + #Puuf = 2.0*(1.0/cnt)*Puuf_cum[:,:,:(np.floor(ds.dims[spectra_dim]/2).astype(int))] ###2.0 is to account for the dropping of the negative side of the FFT + #f = f[:(np.floor(ds.dims[spectra_dim]/2).astype(int))] + Puuf = (1.0/sum_count)*Puuf_cum + f = Pxxf.index.get_level_values('frequency') return f,Puuf def model4D_spatial_spectra(ds,spectra_dim,vert_levels,horizontal_locs,fld,fldMean): From 481c80e954abab8f9b799233ab17cfe844877b95 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 24 Feb 2021 12:15:23 -0700 Subject: [PATCH 023/145] Adding MERRA2 function --- mmctools/wrf/preprocessing.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index b4d967f..033f0b7 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -184,18 +184,8 @@ class MERRA2(RDADataset): Description:https://rda.ucar.edu/datasets/ds313.3/ """ - - def set_resolution(self,resolution_deg=0): - if resolution_deg == 2: - self.res_str = '1.9x2.5' - elif resolution_deg == 1: - self.res_str = '0.9x1.25' - elif resolution_deg == 0.5: - self.res_str = '0.5x0.63' - else: - self.res_str = 'orig_res' - - def download(self,datetimes,path=None): + + def download(self,datetimes,path=None,resolution_deg=0): """Download data at specified datetimes. Files to download: @@ -209,11 +199,22 @@ def download(self,datetimes,path=None): path : str, optional Path to directory in which to save grib files """ + + if resolution_deg == 2: + self.res_str = '1.9x2.5' + elif resolution_deg == 1: + self.res_str = '0.9x1.25' + elif resolution_deg == 0.5: + self.res_str = '0.5x0.63' + else: + self.res_str = 'orig_res' + if path is None: path = '.' else: os.makedirs(path,exist_ok=True) - super().download(urlpath='ds313.3/{}/%Y/MERRA2{}_%Y%m%d.nc'.format(self.res_str), + + super().download(urlpath='ds313.3/'+self.res_str+'/%Y/MERRA2_'+self.res_str+'_%Y%m%d.nc', path=path, datetimes=datetimes) From c02443c3002a3ef7145426a4ff9c60f99668b3a0 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Feb 2021 11:37:23 -0700 Subject: [PATCH 024/145] First draft of a WRF setup script. --- mmctools/wrf/preprocessing.py | 512 ++++++++++++++++++++++++++++++++++ 1 file changed, 512 insertions(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 033f0b7..acc23ce 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -3,6 +3,7 @@ from getpass import getpass import numpy as np import pandas as pd +import glob def prompt(s): if sys.version_info[0] < 3: @@ -381,3 +382,514 @@ def download(self,datetimes,path=None,bounds={}): area=area ) + +class setup_wrf(): + ''' + Set up run directory for WRF / WPS + ''' + + def __init__(self,run_directory,icbc_directory,executables_dict,setup_dict): + self.setup_dict = setup_dict + self.run_directory = run_directory + self.wrf_exe_dir = executables_dict['wrf'] + self.wps_exe_dir = executables_dict['wps'] + self.icbc_dir = icbc_directory + self.icbc_dict = self._get_icbc_info() + missing_namelist_options = self._check_namelist_opts() + if missing_namelist_options != []: + print('missing these namelist options:',missing_namelist_options) + return + + def _get_icbc_info(self): + icbc_type = self.setup_dict['icbc_type'] + interval_seconds = 21600 + met_lvls = 38 + soil_lvls = 4 + download_freq = '6h' + + if icbc_type == 'ERA5': + interval_seconds = 3600 + met_lvls = 38 + download_freq = '1h' + elif icbc_type == 'FNL': + met_lvls = 27 + elif icbc_type == 'NARR': + met_lvls = 30 + elif icbc_type == 'MERRA2': + met_lvls = 73 + download_freq = '1d' + + icbc_dict = {'type' : icbc_type, + 'interval_seconds' : interval_seconds, + 'met_levels' : met_lvls, + 'soil_levels' : soil_lvls, + 'download_freq' : download_freq} + + return icbc_dict + + def _check_namelist_opts(self): + required_fields = ['start_date','end_date','number_of_domains','dxy', + 'time_step','num_eta_levels','parent_grid_ratio', + 'parent_time_ratio','istart','jstart','nx','ny', + 'ref_lat','ref_lon','true_lat1','true_lat2','stand_lon'] + + missing_keys = [] + missing_options = False + setup_keys = list(self.setup_dict.keys()) + for key in required_fields: + if key not in setup_keys: + missing_keys.append(key) + missing_options = True + + if not missing_options: + namelist_defaults = { + 'geogrid_args' : '30s', + 'history_interval' : [60,60], + 'interval_seconds' : self.icbc_dict['interval_seconds'], + 'num_metgrid_levels' : self.icbc_dict['met_levels'], + 'num_metgrid_soil_levels' : self.icbc_dict['soil_levels'], + 'input_from_file' : '.true.', + 'restart_interval' : 360, + 'frames_per_file' : 1, + 'iofields_filename' : 'myoutfields.txt', + 'debug' : 0, + 'ts_locations' : 20, + 'ts_levels' : self.setup_dict['num_eta_levels'], + 'mp_physics' : 10, + 'ra_lw' : 4, + 'ra_sw' : 4, + 'radt' : int(self.setup_dict['dxy']/1000.0), + 'sf_sfclay_physics' : 1, + 'sf_surface_physics' : 2, + 'bl_pbl_physics' : 1, + 'cu_physics' : 1, + 'isfflx' : 1, + 'ifsnow' : 0, + 'icloud' : 0, + 'surface_input_source' : 1, + 'num_soil_layers' : self.icbc_dict['soil_levels'], + 'sf_urban_physics' : 0, + 'w_damping' : 1, + 'diff_opt' : 1, + 'km_opt' : 4, + 'diff_6th_opt' : 2, + 'diff_6th_factor' : 0.12, + 'base_temp' : 290.0, + 'damp_opt' : 3, + 'zdamp' : 5000.0, + 'dampcoef' : 0.2, + 'khdif' : 0, + 'kvdif' : 0, + 'non_hydrostatic' : '.true.', + 'moist_adv_opt' : 1, + 'scalar_adv_opt' : 1, + 'tke_adv_opt' : 1, + 'h_mom_adv_order' : 5, + 'v_mom_adv_order' : 3, + 'h_sca_adv_order' : 5, + 'v_sca_adv_order' : 3, + 'spec_bdy_width' : 5, + 'spec_zone' : 1, + 'relax_zone' : 4, + 'nio_tasks_per_group' : 0, + 'nio_groups' : 1, + } + + namelist_opts = namelist_defaults + for key in setup_keys: + namelist_opts[key] = self.setup_dict[key] + self.namelist_opts = namelist_opts + else: + raise Exception("The following fields are missing from the setup dictionary: ",missing_keys) + + return missing_keys + + def _link_files(self,file_list,destination_dir): + for filen in file_list: + file_name = filen.split('/')[-1] + try: + os.symlink(filen,'{}{}'.format(destination_dir,file_name)) + except FileExistsError: + print('file already linked') + + def link_executables(self): + # Create run dir: + if not os.path.exists(self.run_directory): + os.makedirs(self.run_directory) + + # Link WPS and WRF files / executables + wrf_files = glob.glob('{}[!n]*'.format(self.wrf_exe_dir)) + self._link_files(wrf_files,self.run_directory) + wps_files = glob.glob('{}[!n]*'.format(self.wps_exe_dir)) + self._link_files(wps_files,self.run_directory) + + def _get_nl_str(self,num_doms,phys_opt): + phys_str = '' + for pp in range(0,num_doms): + if type(phys_opt) is list: + phys_str += '{0:>5},'.format(str(phys_opt[pp])) + else: + phys_str += '{0:>5},'.format(str(phys_opt)) + return(phys_str) + + def write_wps_namelist(self): + num_doms = self.namelist_opts['number_of_domains'] + start_date_str = "'{}',".format(self.namelist_opts['start_date'].replace(' ','_'))*num_doms + end_date_str = "'{}',".format(self.namelist_opts['end_date'].replace(' ','_'))*num_doms + geog_data_res = "'{}',".format(self.namelist_opts['geogrid_args'])*num_doms + parent_ids,parent_grid_ratios,dx_str = '','','' + istart_str,jstart_str,nx_str,ny_str = '','','','' + for pp,pgr in enumerate(self.namelist_opts['parent_grid_ratio']): + if pp == 0: + pid = 1 + else: + pid = pp + parent_ids += '{0:>5},'.format(str(pid)) + parent_grid_ratios += '{0:>5},'.format(str(pgr)) + istart_str += '{0:>5},'.format(str(self.namelist_opts['istart'][pp])) + jstart_str += '{0:>5},'.format(str(self.namelist_opts['jstart'][pp])) + nx_str += '{0:>5},'.format(str(self.namelist_opts['nx'][pp])) + ny_str += '{0:>5},'.format(str(self.namelist_opts['ny'][pp])) + f = open('{}namelist.wps'.format(self.run_directory),'w') + f.write("&share\n") + f.write(" wrf_core = 'ARW',\n") + f.write(" max_dom = {},\n".format(num_doms)) + f.write(" start_date = {}\n".format(start_date_str)) + f.write(" end_date = {}\n".format(end_date_str)) + f.write(" interval_seconds = {},\n".format(self.namelist_opts['interval_seconds'])) + f.write(" io_form_geogrid = 2,\n") + f.write("/\n") + f.write("\n") + f.write("&geogrid\n") + f.write(" parent_id = {}\n".format(parent_ids)) + f.write(" parent_grid_ratio = {}\n".format(parent_grid_ratios)) + f.write(" i_parent_start = {}\n".format(istart_str)) + f.write(" j_parent_start = {}\n".format(jstart_str)) + f.write(" e_we = {}\n".format(nx_str)) + f.write(" e_sn = {}\n".format(ny_str)) + f.write(" geog_data_res = {}\n".format(geog_data_res)) + f.write(" dx = {}\n".format(self.namelist_opts['dxy'])) + f.write(" dy = {}\n".format(self.namelist_opts['dxy'])) + f.write(" map_proj = 'lambert',\n") + f.write(" ref_lat = {},\n".format(self.namelist_opts['ref_lat'])) + f.write(" ref_lon = {},\n".format(self.namelist_opts['ref_lon'])) + f.write(" truelat1 = {},\n".format(self.namelist_opts['true_lat1'])) + f.write(" truelat2 = {},\n".format(self.namelist_opts['true_lat2'])) + f.write(" stand_lon = {},\n".format(self.namelist_opts['stand_lon'])) + f.write(" geog_data_path = '/glade/work/hawbecke/geog/',\n") + f.write("/\n") + f.write("\n") + f.write("&ungrib\n") + f.write(" out_format = 'WPS',\n") + f.write(" prefix = '{}',\n".format(self.namelist_opts['icbc_type'].upper())) + f.write("/\n") + f.write("\n") + f.write("&metgrid\n") + f.write(" fg_name = '{}',\n".format(self.namelist_opts['icbc_type'].upper())) + f.write(" io_form_metgrid = 2,\n") + f.write("! constants_name = 'SST:DATE', \n") + f.write("/\n") + f.close() + + + def write_namelist_input(self): + + dt = self.namelist_opts['time_step'] + if (type(dt) is int) or (dt.is_integer()): + ts_0 = int(dt) + ts_num = 0 + ts_den = 1 + else: + ts_0 = 0 + ts_num = dt.as_integer_ratio()[0] + ts_den = dt.as_integer_ratio()[1] + + num_doms = self.namelist_opts['number_of_domains'] + history_interval = self.namelist_opts['history_interval'] + if type(history_interval) is int: + history_interval_str = '{0:>4},'.format(history_interval)*num_doms + elif (type(history_interval) is list) or (type(history_interval) is np.ndarray): + history_interval_str = '' + for hi in history_interval: + history_interval_str += '{0:>4},'.format(hi) + + start_date = pd.to_datetime(self.namelist_opts['start_date']) + end_date = pd.to_datetime(self.namelist_opts['end_date']) + run_hours = int((end_date - start_date).total_seconds()/3600.0) + + start_year_str = "{0:>5},".format(start_date.year)*num_doms + start_month_str = "{0:>5},".format(start_date.month)*num_doms + start_day_str = "{0:>5},".format(start_date.day)*num_doms + start_hour_str = "{0:>5},".format(start_date.hour)*num_doms + start_minute_str = "{0:>5},".format(start_date.minute)*num_doms + start_second_str = "{0:>5},".format(start_date.second)*num_doms + + end_year_str = "{0:>5},".format(end_date.year)*num_doms + end_month_str = "{0:>5},".format(end_date.month)*num_doms + end_day_str = "{0:>5},".format(end_date.day)*num_doms + end_hour_str = "{0:>5},".format(end_date.hour)*num_doms + end_minute_str = "{0:>5},".format(end_date.minute)*num_doms + end_second_str = "{0:>5},".format(end_date.second)*num_doms + + parent_ids,grid_ids,dx_str,radt_str = '','','','' + for pp,pgr in enumerate(self.namelist_opts['parent_grid_ratio']): + grid_ids += '{0:>5},'.format(str(pp+1)) + if pp == 0: + pid = 1 + else: + pid = pp + parent_ids += '{0:>5},'.format(str(pid)) + dx_str += '{0:>5},'.format(str(int(self.namelist_opts['dxy']/pgr))) + radt = self.namelist_opts['radt']/pgr + if radt < 1: radt = 1 + radt_str += '{0:>5},'.format(str(int(radt))) + + grid_ids = self._get_nl_str(num_doms,list(range(1,num_doms+1))) + parent_grid_ratios = self._get_nl_str(num_doms,self.namelist_opts['parent_grid_ratio']) + parent_time_ratios = self._get_nl_str(num_doms,self.namelist_opts['parent_time_ratio']) + istart_str = self._get_nl_str(num_doms,self.namelist_opts['istart']) + jstart_str = self._get_nl_str(num_doms,self.namelist_opts['jstart']) + nx_str = self._get_nl_str(num_doms,self.namelist_opts['nx']) + ny_str = self._get_nl_str(num_doms,self.namelist_opts['ny']) + + io_str = self._get_nl_str(num_doms,self.namelist_opts['iofields_filename']) + mp_str = self._get_nl_str(num_doms,self.namelist_opts['mp_physics']) + sfclay_str = self._get_nl_str(num_doms,self.namelist_opts['sf_sfclay_physics']) + surface_str = self._get_nl_str(num_doms,self.namelist_opts['sf_surface_physics']) + pbl_str = self._get_nl_str(num_doms,self.namelist_opts['bl_pbl_physics']) + cu_str = self._get_nl_str(num_doms,self.namelist_opts['cu_physics']) + urb_str = self._get_nl_str(num_doms,self.namelist_opts['sf_urban_physics']) + diff_str = self._get_nl_str(num_doms,self.namelist_opts['diff_opt']) + km_str = self._get_nl_str(num_doms,self.namelist_opts['km_opt']) + diff6o_str = self._get_nl_str(num_doms,self.namelist_opts['diff_6th_opt']) + diff6f_str = self._get_nl_str(num_doms,self.namelist_opts['diff_6th_factor']) + zdamp_str = self._get_nl_str(num_doms,self.namelist_opts['zdamp']) + damp_str = self._get_nl_str(num_doms,self.namelist_opts['dampcoef']) + khdif_str = self._get_nl_str(num_doms,self.namelist_opts['khdif']) + kvdif_str = self._get_nl_str(num_doms,self.namelist_opts['kvdif']) + nonhyd_str = self._get_nl_str(num_doms,self.namelist_opts['non_hydrostatic']) + moist_str = self._get_nl_str(num_doms,self.namelist_opts['moist_adv_opt']) + scalar_str = self._get_nl_str(num_doms,self.namelist_opts['scalar_adv_opt']) + tke_str = self._get_nl_str(num_doms,self.namelist_opts['tke_adv_opt']) + hmom_str = self._get_nl_str(num_doms,self.namelist_opts['h_mom_adv_order']) + vmom_str = self._get_nl_str(num_doms,self.namelist_opts['v_mom_adv_order']) + hsca_str = self._get_nl_str(num_doms,self.namelist_opts['h_sca_adv_order']) + vsca_str = self._get_nl_str(num_doms,self.namelist_opts['v_sca_adv_order']) + + specified = ['.false.']*num_doms + nested = ['.true.']*num_doms + specified[0] = '.true.' + nested[0] = '.false.' + + f = open('{}namelist.input'.format(self.run_directory),'w') + f.write("&time_control\n") + f.write(" run_days = 0,\n") + f.write(" run_hours = {0:>5},\n".format(run_hours)) + f.write(" run_minutes = 0,\n") + f.write(" run_seconds = 0,\n") + f.write(" start_year = {}\n".format(start_year_str)) + f.write(" start_month = {}\n".format(start_month_str)) + f.write(" start_day = {}\n".format(start_day_str)) + f.write(" start_hour = {}\n".format(start_hour_str)) + f.write(" start_minute = {}\n".format(start_minute_str)) + f.write(" start_second = {}\n".format(start_second_str)) + f.write(" end_year = {}\n".format(end_year_str)) + f.write(" end_month = {}\n".format(end_month_str)) + f.write(" end_day = {}\n".format(end_day_str)) + f.write(" end_hour = {}\n".format(end_hour_str)) + f.write(" end_minute = {}\n".format(end_minute_str)) + f.write(" end_second = {}\n".format(end_second_str)) + f.write(" interval_seconds = {},\n".format(self.namelist_opts['interval_seconds'])) + f.write(" input_from_file = {}\n".format("{},".format(self.namelist_opts['input_from_file'])*num_doms)) + f.write(" restart = .false.,\n") + f.write(" restart_interval = {},\n".format(self.namelist_opts['restart_interval'])) + f.write(" io_form_history = 2\n") + f.write(" io_form_restart = 2\n") + f.write(" io_form_input = 2\n") + f.write(" io_form_boundary = 2\n") + f.write(" history_interval = {}\n".format(history_interval_str)) + f.write(" frames_per_outfile = {}\n".format("{0:>5},".format(self.namelist_opts['frames_per_file'])*num_doms)) + f.write(" iofields_filename = {}\n".format(io_str)) + f.write(" ignore_iofields_warning = .true.,\n") + f.write(" debug_level = {} \n".format(self.namelist_opts['debug'])) + f.write("/\n") + f.write("\n") + f.write("&domains\n") + f.write(" time_step = {},\n".format(ts_0)) + f.write(" time_step_fract_num = {},\n".format(ts_num)) + f.write(" time_step_fract_den = {},\n".format(ts_den)) + f.write(" max_dom = {},\n".format(num_doms)) + f.write(" max_ts_locs = {},\n".format(self.namelist_opts['ts_locations'])) + f.write(" max_ts_level = {},\n".format(self.namelist_opts['ts_locations'])) + f.write(" tslist_unstagger_winds = .true., \n") + f.write(" s_we = {}\n".format("{0:>5},".format(1)*num_doms)) + f.write(" e_we = {}\n".format(nx_str)) + f.write(" s_sn = {}\n".format("{0:>5},".format(1)*num_doms)) + f.write(" e_sn = {}\n".format(ny_str)) + f.write(" s_vert = {}\n".format("{0:>5},".format(1)*num_doms)) + f.write(" e_vert = {}\n".format("{0:>5},".format(self.namelist_opts['num_eta_levels'])*num_doms)) + f.write(" eta_levels = {},\n".format(self.namelist_opts['eta_levels'])) + f.write(" p_top_requested = {},\n".format(self.namelist_opts['p_top_requested'])) + f.write(" num_metgrid_levels = {},\n".format(self.namelist_opts['num_metgrid_levels'])) + f.write(" num_metgrid_soil_levels = {},\n".format(self.namelist_opts['num_metgrid_soil_levels'])) + f.write(" dx = {}\n".format(dx_str)) + f.write(" dy = {}\n".format(dx_str)) + f.write(" grid_id = {}\n".format(grid_ids)) + f.write(" parent_id = {}\n".format(parent_ids)) + f.write(" i_parent_start = {}\n".format(istart_str)) + f.write(" j_parent_start = {}\n".format(jstart_str)) + f.write(" parent_grid_ratio = {}\n".format(parent_grid_ratios)) + f.write(" parent_time_step_ratio = {}\n".format(parent_time_ratios)) + f.write(" feedback = {},\n".format(self.namelist_opts['feedback'])) + f.write(" smooth_option = {},\n".format(self.namelist_opts['smooth_option'])) + f.write(" /\n") + f.write("\n") + f.write("&physics\n") + f.write(" mp_physics = {}\n".format(mp_str)) + f.write(" ra_lw_physics = {}\n".format("{0:>5},".format(self.namelist_opts['ra_lw'])*num_doms)) + f.write(" ra_sw_physics = {}\n".format("{0:>5},".format(self.namelist_opts['ra_sw'])*num_doms)) + f.write(" radt = {}\n".format(radt_str)) + f.write(" sf_sfclay_physics = {}\n".format(sfclay_str)) + f.write(" sf_surface_physics = {}\n".format(surface_str)) + f.write(" bl_pbl_physics = {}\n".format(pbl_str)) + f.write(" bldt = {}\n".format("{0:>5},".format(0)*num_doms)) + f.write(" cu_physics = {}\n".format(cu_str)) + f.write(" cudt = {}\n".format("{0:>5},".format(5)*num_doms)) + f.write(" isfflx = {}, \n".format(self.namelist_opts['isfflx'])) + f.write(" ifsnow = {}, \n".format(self.namelist_opts['ifsnow'])) + f.write(" icloud = {}, \n".format(self.namelist_opts['icloud'])) + f.write(" surface_input_source = {}, \n".format(self.namelist_opts['surface_input_source'])) + f.write(" num_soil_layers = {}, \n".format(self.namelist_opts['num_soil_layers'])) + f.write(" sf_urban_physics = {}\n".format(urb_str)) + f.write(" /\n") + f.write("\n") + f.write("&fdda\n") + f.write("/\n") + f.write("\n") + f.write("&dynamics\n") + f.write(" w_damping = {}, \n".format(self.namelist_opts['w_damping'])) + f.write(" diff_opt = {}\n".format(diff_str)) + f.write(" km_opt = {}\n".format(km_str)) + f.write(" diff_6th_opt = {}\n".format(diff6o_str)) + f.write(" diff_6th_factor = {}\n".format(diff6f_str)) + f.write(" base_temp = {}, \n".format(self.namelist_opts['base_temp'])) + f.write(" damp_opt = {}, \n".format(self.namelist_opts['damp_opt'])) + f.write(" zdamp = {}\n".format(zdamp_str)) + f.write(" dampcoef = {}\n".format(damp_str)) + f.write(" khdif = {}\n".format(khdif_str)) + f.write(" kvdif = {}\n".format(kvdif_str)) + f.write(" non_hydrostatic = {}\n".format(nonhyd_str)) + f.write(" moist_adv_opt = {}\n".format(moist_str)) + f.write(" scalar_adv_opt = {}\n".format(scalar_str)) + f.write(" tke_adv_opt = {}\n".format(tke_str)) + f.write(" h_mom_adv_order = {}\n".format(hmom_str)) + f.write(" v_mom_adv_order = {}\n".format(vmom_str)) + f.write(" h_sca_adv_order = {}\n".format(hsca_str)) + f.write(" v_sca_adv_order = {}\n".format(vsca_str)) + f.write(" /\n") + f.write(" \n") + f.write("&bdy_control\n") + f.write(" spec_bdy_width = {}, \n".format(self.namelist_opts['spec_bdy_width'])) + f.write(" spec_zone = {}, \n".format(self.namelist_opts['spec_zone'])) + f.write(" relax_zone = {}, \n".format(self.namelist_opts['relax_zone'])) + f.write(" specified = {}, \n".format(','.join(specified))) + f.write(" nested = {}, \n".format(','.join(nested))) + f.write("/\n") + f.write("\n") + f.write(" &namelist_quilt\n") + f.write(" nio_tasks_per_group = {}, \n".format(self.namelist_opts['nio_tasks_per_group'])) + f.write(" nio_groups = {}, \n".format(self.namelist_opts['nio_groups'])) + f.write(" /\n") + + def get_icbcs(self): + start_time = pd.to_datetime(self.setup_dict['start_date']) + end_time = pd.to_datetime(self.setup_dict['end_date']) + freq = self.icbc_dict['download_freq'] + + datetimes = pd.date_range(start=start_time, + end=end_time, + freq=freq) + optional_args = {} + icbc_type = self.namelist_opts['icbc_type'].upper() + if icbc_type == 'ERAI': + icbc = ERAInterim() + elif icbc_type == 'FNL': + icbc = FNL() + elif icbc_type == 'MERRA2': + icbc = MERRA2() + if 'resolution_deg' not in self.setup_keys(): + res_drag = 0 + else: + res_drag = self.setup_dict['resolution_deg'] + optional_args['resolution_deg'] = res_drag + elif icbc_type == 'ERA5': + icbc = ERA5() + if 'bounds' not in self.setup_dict.keys(): + bounds = { 'N':60, + 'S':30, + 'W':-120, + 'E':-90} + else: + bounds = self.setup_dict['bounds'] + optional_args['bounds'] = bounds + else: + print('We currently do not support ',icbc_type) + + icbc.download(datetimes,path=self.icbc_dir, **optional_args) + + + def write_submission_scripts(self,submission_dict,executable,hpc='cheyenne'): + + if hpc == 'cheyenne': + f = open('{}submit_{}.sh'.format(self.run_directory,executable),'w') + f.write("#!/bin/bash\n") + run_str = '{0}{1}'.format(self.icbc_dict['type'], + (self.setup_dict['start_date'].split(' ')[0]).replace('-','')) + f.write("#PBS -N {} \n".format(run_str)) + f.write("#PBS -A {}\n".format(submission_dict['account_key'])) + f.write("#PBS -l walltime={0:02d}:00:00\n".format(submission_dict['walltime_hours'][executable])) + f.write("#PBS -q economy\n") + f.write("#PBS -j oe\n") + f.write("#PBS -m abe\n") + f.write("#PBS -M {}\n".format(submission_dict['user_email'])) + f.write("### Select 2 nodes with 36 CPUs each for a total of 72 MPI processes\n") + if executable == 'wps': + f.write("#PBS -l select=1:ncpus=1:mpiprocs=1\n".format(submission_dict['nodes'])) + else: + f.write("#PBS -l select={0:02d}:ncpus=36:mpiprocs=36\n".format(submission_dict['nodes'])) + f.write("date_start=`date`\n") + f.write("echo $date_start\n") + f.write("module list\n") + if executable == 'wps': + icbc_type = self.icbc_dict['type'].upper() + if icbc_type == 'ERA5': + icbc_head = 'era5_*' + icbc_vtable = 'ERA-interim.pl' + elif icbc_type == 'ERAI': + icbc_head = 'ei.oper*' + icbc_vtable = 'ERA-interim.pl' + elif icbc_type == 'FNL': + icbc_head = 'fnl_*' + icbc_vtable = 'GFS' + elif icbc_type == 'MERRA2': + icbc_head = 'MERRA2_*' + icbc_vtable = 'GFS' + else: + print('We do not support this ICBC yet...') + + icbc_files = '{}{}'.format(self.icbc_dir,icbc_head) + f.write("./link_grib.csh {}\n".format(icbc_files)) + f.write("ln -sf ungrib/Variable_Tables/Vtable.{} Vtable\n".format(icbc_vtable)) + f.write("./geogrid.exe\n".format(executable)) + f.write("./ungrib.exe\n".format(executable)) + f.write("./metgrid.exe\n".format(executable)) + f.write("for i in GRIBFILE.*; do unlink $i; done\n") + else: + f.write("mpiexec_mpt ./{}.exe\n".format(executable)) + f.write("date_end=`date`\n") + f.write("echo $date_end\n") + f.close() + + else: + print('The hpc requested, {}, is not currently supported... please add it!'.format(hpc)) + From 693b506cb876474a19d12beae5875e524b714fb7 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Feb 2021 13:38:13 -0700 Subject: [PATCH 025/145] Need option for different landuse datasets --- mmctools/wrf/preprocessing.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index acc23ce..5bf1742 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -412,7 +412,7 @@ def _get_icbc_info(self): met_lvls = 38 download_freq = '1h' elif icbc_type == 'FNL': - met_lvls = 27 + met_lvls = 34 elif icbc_type == 'NARR': met_lvls = 30 elif icbc_type == 'MERRA2': @@ -442,6 +442,10 @@ def _check_namelist_opts(self): missing_options = True if not missing_options: + if 'usgs' in self.setup_dict['geogrid_args']: + land_cat = 24 + else: + land_cat = 21 namelist_defaults = { 'geogrid_args' : '30s', 'history_interval' : [60,60], @@ -468,6 +472,7 @@ def _check_namelist_opts(self): 'icloud' : 0, 'surface_input_source' : 1, 'num_soil_layers' : self.icbc_dict['soil_levels'], + 'num_land_cat' : land_cat, 'sf_urban_physics' : 0, 'w_damping' : 1, 'diff_opt' : 1, @@ -760,6 +765,7 @@ def write_namelist_input(self): f.write(" icloud = {}, \n".format(self.namelist_opts['icloud'])) f.write(" surface_input_source = {}, \n".format(self.namelist_opts['surface_input_source'])) f.write(" num_soil_layers = {}, \n".format(self.namelist_opts['num_soil_layers'])) + f.write(" num_land_cat = {}, \n".format(self.namelist_opts['num_land_cat'])) f.write(" sf_urban_physics = {}\n".format(urb_str)) f.write(" /\n") f.write("\n") From 6231cba1f4b341a885a4cb9e443c71f32e059bbd Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Feb 2021 14:30:06 -0700 Subject: [PATCH 026/145] Adding io_fields_filename write capability --- mmctools/wrf/preprocessing.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 5bf1742..7523144 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -899,3 +899,46 @@ def write_submission_scripts(self,submission_dict,executable,hpc='cheyenne'): else: print('The hpc requested, {}, is not currently supported... please add it!'.format(hpc)) + + + def write_io_fieldnames(self,vars_to_remove,vars_to_add): + io_names = self.setup_dict['iofields_filename'] + + if type(io_names) is str: + io_names = [io_names] + if type(io_names) is not list: + io_names = list(io_names) + + assert (len(vars_to_remove) == len(vars_to_add)) and (len(vars_to_add) == len(np.unique(io_names))), \ + 'expecting number of io field names ({}) and add/remove lists ({}/{}) to be same shape'.format( + len(np.unique(io_names)),len(vars_to_add),len(vars_to_remove)) + + rem_str_start = '-:h:0:' + add_str_start = '+:h:0:' + + for ii,io_name in enumerate(np.unique(io_names)): + rem_vars = vars_to_remove[ii] + add_vars = vars_to_add[ii] + f = open('{}{}'.format(self.run_directory,io_name),'w') + line = '' + var_count = 0 + for rv in rem_vars: + line += '{},'.format(rv) + if var_count == 7: + f.write('{}{}\n'.format(rem_str_start,line)) + var_count = 0 + line = '' + else: + var_count += 1 + + var_count = 0 + for av in add_vars: + line += '{},'.format(av) + if var_count == 7: + f.write('{}{}\n'.format(add_str_start,line)) + var_count = 0 + line = '' + else: + var_count += 1 + + f.close() \ No newline at end of file From 6784a860d1988763710aa0ac812f8dd183611127 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Feb 2021 14:49:12 -0700 Subject: [PATCH 027/145] Removing iofields_filename requirement in namelist --- mmctools/wrf/preprocessing.py | 74 +++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 7523144..688a95a 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -455,7 +455,6 @@ def _check_namelist_opts(self): 'input_from_file' : '.true.', 'restart_interval' : 360, 'frames_per_file' : 1, - 'iofields_filename' : 'myoutfields.txt', 'debug' : 0, 'ts_locations' : 20, 'ts_levels' : self.setup_dict['num_eta_levels'], @@ -656,8 +655,12 @@ def write_namelist_input(self): jstart_str = self._get_nl_str(num_doms,self.namelist_opts['jstart']) nx_str = self._get_nl_str(num_doms,self.namelist_opts['nx']) ny_str = self._get_nl_str(num_doms,self.namelist_opts['ny']) - - io_str = self._get_nl_str(num_doms,self.namelist_opts['iofields_filename']) + + if 'iofields_filename' in self.namelist_opts.keys(): + include_io = True + io_str = self._get_nl_str(num_doms,self.namelist_opts['iofields_filename']) + else: + include_io = False mp_str = self._get_nl_str(num_doms,self.namelist_opts['mp_physics']) sfclay_str = self._get_nl_str(num_doms,self.namelist_opts['sf_sfclay_physics']) surface_str = self._get_nl_str(num_doms,self.namelist_opts['sf_surface_physics']) @@ -714,8 +717,9 @@ def write_namelist_input(self): f.write(" io_form_boundary = 2\n") f.write(" history_interval = {}\n".format(history_interval_str)) f.write(" frames_per_outfile = {}\n".format("{0:>5},".format(self.namelist_opts['frames_per_file'])*num_doms)) - f.write(" iofields_filename = {}\n".format(io_str)) - f.write(" ignore_iofields_warning = .true.,\n") + if include_io: + f.write(" iofields_filename = {}\n".format(io_str)) + f.write(" ignore_iofields_warning = .true.,\n") f.write(" debug_level = {} \n".format(self.namelist_opts['debug'])) f.write("/\n") f.write("\n") @@ -901,7 +905,10 @@ def write_submission_scripts(self,submission_dict,executable,hpc='cheyenne'): - def write_io_fieldnames(self,vars_to_remove,vars_to_add): + def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): + if 'iofields_filename' not in self.setup_dict.keys(): + print('iofields_filename not found in setup dict... add a name to allow for creating the file') + return io_names = self.setup_dict['iofields_filename'] if type(io_names) is str: @@ -909,36 +916,43 @@ def write_io_fieldnames(self,vars_to_remove,vars_to_add): if type(io_names) is not list: io_names = list(io_names) - assert (len(vars_to_remove) == len(vars_to_add)) and (len(vars_to_add) == len(np.unique(io_names))), \ - 'expecting number of io field names ({}) and add/remove lists ({}/{}) to be same shape'.format( - len(np.unique(io_names)),len(vars_to_add),len(vars_to_remove)) + if vars_to_remove is not None: + assert len(vars_to_remove) == len(np.unique(io_names)), \ + 'expecting number of io field names ({}) and remove lists {} to be same shape'.format( + len(np.unique(io_names)),len(vars_to_remove)) + if vars_to_add is not None: + assert len(vars_to_add) == len(np.unique(io_names)), \ + 'expecting number of io field names ({}) and add lists {} to be same shape'.format( + len(np.unique(io_names)),len(vars_to_add)) rem_str_start = '-:h:0:' add_str_start = '+:h:0:' for ii,io_name in enumerate(np.unique(io_names)): - rem_vars = vars_to_remove[ii] - add_vars = vars_to_add[ii] f = open('{}{}'.format(self.run_directory,io_name),'w') line = '' - var_count = 0 - for rv in rem_vars: - line += '{},'.format(rv) - if var_count == 7: - f.write('{}{}\n'.format(rem_str_start,line)) - var_count = 0 - line = '' - else: - var_count += 1 - - var_count = 0 - for av in add_vars: - line += '{},'.format(av) - if var_count == 7: - f.write('{}{}\n'.format(add_str_start,line)) - var_count = 0 - line = '' - else: - var_count += 1 + if vars_to_remove is not None: + rem_vars = vars_to_remove[ii] + var_count = 0 + for rv in rem_vars: + line += '{},'.format(rv) + if var_count == 7: + f.write('{}{}\n'.format(rem_str_start,line)) + var_count = 0 + line = '' + else: + var_count += 1 + + if vars_to_add is not None: + add_vars = vars_to_add[ii] + var_count = 0 + for av in add_vars: + line += '{},'.format(av) + if var_count == 7: + f.write('{}{}\n'.format(add_str_start,line)) + var_count = 0 + line = '' + else: + var_count += 1 f.close() \ No newline at end of file From d3fa1cfdba1351977be292b866096ca8a8671662 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 2 Mar 2021 08:31:42 -0700 Subject: [PATCH 028/145] Remove eta_levels from requirements/add submit_all Eta levels do not need to be specified - you can let WRF do that on its own if you give the number of levels. Added scripts that will be in the main directory to submit all wps, real, and wrf submission scripts. For running with several cases, this is extremely useful to just kick off all at once. --- mmctools/wrf/preprocessing.py | 128 ++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 54 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 688a95a..95f8494 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -737,7 +737,8 @@ def write_namelist_input(self): f.write(" e_sn = {}\n".format(ny_str)) f.write(" s_vert = {}\n".format("{0:>5},".format(1)*num_doms)) f.write(" e_vert = {}\n".format("{0:>5},".format(self.namelist_opts['num_eta_levels'])*num_doms)) - f.write(" eta_levels = {},\n".format(self.namelist_opts['eta_levels'])) + if 'eta_levels' in self.namelist_opts.keys(): + f.write(" eta_levels = {},\n".format(self.namelist_opts['eta_levels'])) f.write(" p_top_requested = {},\n".format(self.namelist_opts['p_top_requested'])) f.write(" num_metgrid_levels = {},\n".format(self.namelist_opts['num_metgrid_levels'])) f.write(" num_metgrid_soil_levels = {},\n".format(self.namelist_opts['num_metgrid_soil_levels'])) @@ -848,60 +849,61 @@ def get_icbcs(self): icbc.download(datetimes,path=self.icbc_dir, **optional_args) - def write_submission_scripts(self,submission_dict,executable,hpc='cheyenne'): - - if hpc == 'cheyenne': - f = open('{}submit_{}.sh'.format(self.run_directory,executable),'w') - f.write("#!/bin/bash\n") - run_str = '{0}{1}'.format(self.icbc_dict['type'], - (self.setup_dict['start_date'].split(' ')[0]).replace('-','')) - f.write("#PBS -N {} \n".format(run_str)) - f.write("#PBS -A {}\n".format(submission_dict['account_key'])) - f.write("#PBS -l walltime={0:02d}:00:00\n".format(submission_dict['walltime_hours'][executable])) - f.write("#PBS -q economy\n") - f.write("#PBS -j oe\n") - f.write("#PBS -m abe\n") - f.write("#PBS -M {}\n".format(submission_dict['user_email'])) - f.write("### Select 2 nodes with 36 CPUs each for a total of 72 MPI processes\n") - if executable == 'wps': - f.write("#PBS -l select=1:ncpus=1:mpiprocs=1\n".format(submission_dict['nodes'])) - else: - f.write("#PBS -l select={0:02d}:ncpus=36:mpiprocs=36\n".format(submission_dict['nodes'])) - f.write("date_start=`date`\n") - f.write("echo $date_start\n") - f.write("module list\n") - if executable == 'wps': - icbc_type = self.icbc_dict['type'].upper() - if icbc_type == 'ERA5': - icbc_head = 'era5_*' - icbc_vtable = 'ERA-interim.pl' - elif icbc_type == 'ERAI': - icbc_head = 'ei.oper*' - icbc_vtable = 'ERA-interim.pl' - elif icbc_type == 'FNL': - icbc_head = 'fnl_*' - icbc_vtable = 'GFS' - elif icbc_type == 'MERRA2': - icbc_head = 'MERRA2_*' - icbc_vtable = 'GFS' + def write_submission_scripts(self,submission_dict,hpc='cheyenne'): + executables = ['wps','real','wrf'] + for executable in executables: + if hpc == 'cheyenne': + f = open('{}submit_{}.sh'.format(self.run_directory,executable),'w') + f.write("#!/bin/bash\n") + run_str = '{0}{1}'.format(self.icbc_dict['type'], + (self.setup_dict['start_date'].split(' ')[0]).replace('-','')) + f.write("#PBS -N {} \n".format(run_str)) + f.write("#PBS -A {}\n".format(submission_dict['account_key'])) + f.write("#PBS -l walltime={0:02d}:00:00\n".format(submission_dict['walltime_hours'][executable])) + f.write("#PBS -q economy\n") + f.write("#PBS -j oe\n") + f.write("#PBS -m abe\n") + f.write("#PBS -M {}\n".format(submission_dict['user_email'])) + f.write("### Select 2 nodes with 36 CPUs each for a total of 72 MPI processes\n") + if executable == 'wps': + f.write("#PBS -l select=1:ncpus=1:mpiprocs=1\n".format(submission_dict['nodes'])) else: - print('We do not support this ICBC yet...') - - icbc_files = '{}{}'.format(self.icbc_dir,icbc_head) - f.write("./link_grib.csh {}\n".format(icbc_files)) - f.write("ln -sf ungrib/Variable_Tables/Vtable.{} Vtable\n".format(icbc_vtable)) - f.write("./geogrid.exe\n".format(executable)) - f.write("./ungrib.exe\n".format(executable)) - f.write("./metgrid.exe\n".format(executable)) - f.write("for i in GRIBFILE.*; do unlink $i; done\n") + f.write("#PBS -l select={0:02d}:ncpus=36:mpiprocs=36\n".format(submission_dict['nodes'])) + f.write("date_start=`date`\n") + f.write("echo $date_start\n") + f.write("module list\n") + if executable == 'wps': + icbc_type = self.icbc_dict['type'].upper() + if icbc_type == 'ERA5': + icbc_head = 'era5_*' + icbc_vtable = 'ERA-interim.pl' + elif icbc_type == 'ERAI': + icbc_head = 'ei.oper*' + icbc_vtable = 'ERA-interim.pl' + elif icbc_type == 'FNL': + icbc_head = 'fnl_*' + icbc_vtable = 'GFS' + elif icbc_type == 'MERRA2': + icbc_head = 'MERRA2_*' + icbc_vtable = 'GFS' + else: + print('We do not support this ICBC yet...') + + icbc_files = '{}{}'.format(self.icbc_dir,icbc_head) + f.write("./link_grib.csh {}\n".format(icbc_files)) + f.write("ln -sf ungrib/Variable_Tables/Vtable.{} Vtable\n".format(icbc_vtable)) + f.write("./geogrid.exe\n".format(executable)) + f.write("./ungrib.exe\n".format(executable)) + f.write("./metgrid.exe\n".format(executable)) + f.write("for i in GRIBFILE.*; do unlink $i; done\n") + else: + f.write("mpiexec_mpt ./{}.exe\n".format(executable)) + f.write("date_end=`date`\n") + f.write("echo $date_end\n") + f.close() + else: - f.write("mpiexec_mpt ./{}.exe\n".format(executable)) - f.write("date_end=`date`\n") - f.write("echo $date_end\n") - f.close() - - else: - print('The hpc requested, {}, is not currently supported... please add it!'.format(hpc)) + print('The hpc requested, {}, is not currently supported... please add it!'.format(hpc)) @@ -955,4 +957,22 @@ def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): else: var_count += 1 - f.close() \ No newline at end of file + f.close() + + + + def create_submitAll_scripts(self,main_directory,list_of_cases,executables): + str_of_dirs = ' '.join(list_of_cases) + for exe in executables: + fname = '{}submit_all_{}.sh'.format(main_directory,exe) + f = open(fname,'w') + f.write("#!/bin/bash\n") + f.write("for value in {}\n".format(str_of_dirs)) + f.write("do\n") + f.write(" cd $value/\n") + f.write(" pwd\n") + f.write(" qsub submit_{}.sh\n".format(exe)) + f.write(" cd ..\n") + f.write("done\n") + f.close() + os.chmod(fname,0o755) \ No newline at end of file From 3ed37acb8a3102540e92fc3ffa7132f817dfcb1b Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 2 Mar 2021 09:20:06 -0700 Subject: [PATCH 029/145] Adding tslist file, cleanup var names Allow for printing a tslist file with specified lat/lon or i/j locations. Change variable name from 'run_directory' to 'run_dir' within the setup_wrf class to match format of other directory variable names --- mmctools/wrf/preprocessing.py | 114 ++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 12 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 95f8494..7871b51 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -167,6 +167,7 @@ def download(self,datetimes,path=None): else: os.makedirs(path,exist_ok=True) # pressure-level data + super().download(urlpath='ds627.0/{prefix:s}/%Y%m/{prefix:s}.{field:s}.%Y%m%d%H', path=path, prefix='ei.oper.an.pl', @@ -390,7 +391,7 @@ class setup_wrf(): def __init__(self,run_directory,icbc_directory,executables_dict,setup_dict): self.setup_dict = setup_dict - self.run_directory = run_directory + self.run_dir = run_directory self.wrf_exe_dir = executables_dict['wrf'] self.wps_exe_dir = executables_dict['wps'] self.icbc_dir = icbc_directory @@ -518,14 +519,14 @@ def _link_files(self,file_list,destination_dir): def link_executables(self): # Create run dir: - if not os.path.exists(self.run_directory): - os.makedirs(self.run_directory) + if not os.path.exists(self.run_dir): + os.makedirs(self.run_dir) # Link WPS and WRF files / executables wrf_files = glob.glob('{}[!n]*'.format(self.wrf_exe_dir)) - self._link_files(wrf_files,self.run_directory) + self._link_files(wrf_files,self.run_dir) wps_files = glob.glob('{}[!n]*'.format(self.wps_exe_dir)) - self._link_files(wps_files,self.run_directory) + self._link_files(wps_files,self.run_dir) def _get_nl_str(self,num_doms,phys_opt): phys_str = '' @@ -554,7 +555,7 @@ def write_wps_namelist(self): jstart_str += '{0:>5},'.format(str(self.namelist_opts['jstart'][pp])) nx_str += '{0:>5},'.format(str(self.namelist_opts['nx'][pp])) ny_str += '{0:>5},'.format(str(self.namelist_opts['ny'][pp])) - f = open('{}namelist.wps'.format(self.run_directory),'w') + f = open('{}namelist.wps'.format(self.run_dir),'w') f.write("&share\n") f.write(" wrf_core = 'ARW',\n") f.write(" max_dom = {},\n".format(num_doms)) @@ -689,7 +690,7 @@ def write_namelist_input(self): specified[0] = '.true.' nested[0] = '.false.' - f = open('{}namelist.input'.format(self.run_directory),'w') + f = open('{}namelist.input'.format(self.run_dir),'w') f.write("&time_control\n") f.write(" run_days = 0,\n") f.write(" run_hours = {0:>5},\n".format(run_hours)) @@ -845,7 +846,7 @@ def get_icbcs(self): optional_args['bounds'] = bounds else: print('We currently do not support ',icbc_type) - + icbc.download(datetimes,path=self.icbc_dir, **optional_args) @@ -853,7 +854,7 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): executables = ['wps','real','wrf'] for executable in executables: if hpc == 'cheyenne': - f = open('{}submit_{}.sh'.format(self.run_directory,executable),'w') + f = open('{}submit_{}.sh'.format(self.run_dir,executable),'w') f.write("#!/bin/bash\n") run_str = '{0}{1}'.format(self.icbc_dict['type'], (self.setup_dict['start_date'].split(' ')[0]).replace('-','')) @@ -931,7 +932,7 @@ def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): add_str_start = '+:h:0:' for ii,io_name in enumerate(np.unique(io_names)): - f = open('{}{}'.format(self.run_directory,io_name),'w') + f = open('{}{}'.format(self.run_dir,io_name),'w') line = '' if vars_to_remove is not None: rem_vars = vars_to_remove[ii] @@ -960,7 +961,6 @@ def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): f.close() - def create_submitAll_scripts(self,main_directory,list_of_cases,executables): str_of_dirs = ' '.join(list_of_cases) for exe in executables: @@ -975,4 +975,94 @@ def create_submitAll_scripts(self,main_directory,list_of_cases,executables): f.write(" cd ..\n") f.write("done\n") f.close() - os.chmod(fname,0o755) \ No newline at end of file + os.chmod(fname,0o755) + + def create_tslist_file(self,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): + fname = '{}tslist'.format(self.run_dir) + write_tslist_file(fname,lat=lat,lon=lon,i=i,j=j,twr_names=twr_names,twr_abbr=twr_abbr) + + +def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): + """ + Write a list of lat/lon or i/j locations to a tslist file that is + readable by WRF. + + Usage + ==== + fname : string + The path to and filename of the file to be created + lat,lon,i,j : list or 1-D array + Locations of the towers. + If using lat/lon - locx = lon, locy = lat + If using i/j - locx = i, locy = j + twr_names : list of strings, optional + List of names for each tower location. Names should not be + longer than 25 characters, each. If None, default names will + be given. + twr_abbr : list of strings, optional + List of abbreviations for each tower location. Names should not be + longer than 5 characters, each. If None, default abbreviations + will be given. + """ + if (lat is not None) and (lon is not None) and (i is None) and (j is None): + header_keys = '# 24 characters for name | pfx | LAT | LON |' + twr_locx = lon + twr_locy = lat + ij_or_ll = 'll' + elif (i is not None) and (j is not None) and (lat is None) and (lon is None): + header_keys = '# 24 characters for name | pfx | I | J |' + twr_locx = i + twr_locy = j + ij_or_ll = 'ij' + else: + print('Please specify either lat&lon or i&j') + return + + header_line = '#-----------------------------------------------#' + header = '{}\n{}\n{}\n'.format(header_line,header_keys,header_line) + + if len(twr_locy) == len(twr_locx): + ntowers = len(twr_locy) + else: + print('Error - tower_x: {}, tower_y: {}'.format(len(twr_locx),len(twr_locy))) + return + + if not isinstance(twr_names,list): + twr_names = list(twr_names) + if twr_names != None: + if len(twr_names) != ntowers: + print('Error - Tower names: {}, tower_x: {}, tower_y: {}'.format(len(twr_names),len(twr_locx),len(twr_locy))) + return + else: + twr_names = [] + for twr in np.arange(0,ntowers): + twr_names.append('Tower{0:04d}'.format(twr+1)) + + if not isinstance(twr_abbr,list): + twr_abbr = list(twr_abbr) + if twr_abbr != None: + if len(twr_abbr) != ntowers: + print('Error - Tower abbr: {}, tower_x: {}, tower_y: {}'.format(len(twr_abbr),len(twr_locx),len(twr_locy))) + return + if len(max(twr_abbr,key=len)) > 5: + print('Tower abbreviations are too large... setting to default names') + twr_abbr = None + if twr_abbr==None: + twr_abbr = [] + for twr in np.arange(0,ntowers): + twr_abbr.append('T{0:04d}'.format(twr+1)) + + f = open(fname,'w') + f.write(header) + + for tt in range(0,ntowers): + if ij_or_ll == 'ij': + twr_line = '{0:<26.25}{1: <6}{2: <8d} {3: <8d}\n'.format( + twr_names[tt], twr_abbr[tt], int(twr_locx[tt]), int(twr_locy[tt])) + else: + twr_line = '{0:<26.25}{1: <6}{2:.7s} {3:<.8s}\n'.format( + twr_names[tt], twr_abbr[tt], '{0:8.7f}'.format(float(twr_locy[tt])), + '{0:8.7f}'.format(float(twr_locx[tt]))) + f.write(twr_line) + f.close() + \ No newline at end of file From 3bbfe200c43b40b82d39c24d5bd1c7d705a319f1 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Mar 2021 12:59:29 -0700 Subject: [PATCH 030/145] Add temp_var option to wrfout_seriesReader() --- mmctools/wrf/utils.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 8fed1c2..86f1675 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1140,7 +1140,7 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, - hlim_ind=None): + hlim_ind=None,temp_var='THM'): """ Construct an a2e-mmc standard, xarrays-based, data structure from a series of 3-dimensional WRF output files @@ -1157,15 +1157,18 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, output files. specified_heights : list-like, optional If not None, then a list of static heights to which all data - variables should be interpolated. Note that this significantly + variables should be interpolated. Note that this significantly increases the data read time. hlim_ind : int, index - If not none, then the DataArray ds_subset is further subset by vertical dimension, - keeping vertical layers 0:hlim_ind. - This is meant to be used to speed up execution of the code or prevent a memory error - where the specified_heights argument is not well suited - (i.e., want a range of non-interpolated heights), and you only care about - data that are below a certain vertical index. + If not none, then the DataArray ds_subset is further subset by + vertical dimension, keeping vertical layers 0:hlim_ind. This is + meant to be used to speed up execution of the code or prevent a + memory error where the specified_heights argument is not well + suited (i.e., want a range of non-interpolated heights), and you + only care about data that are below a certain vertical index. + temp_var : str, optional + Name of moist potential temperature variable, e.g., 'THM' for + standard WRF output or 'T' MMC auxiliary output """ import wrf as wrfpy TH0 = 300.0 #WRF convention base-state theta = 300.0 K @@ -1212,7 +1215,7 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, print('Extracting data variables, p,theta...') ds_subset['p'] = xr.DataArray(ds['P']+ds['PB'], dims=dim_keys) - ds_subset['theta'] = xr.DataArray(ds['THM']+TH0, dims=dim_keys) + ds_subset['theta'] = xr.DataArray(ds[temp_var]+TH0, dims=dim_keys) # optionally, interpolate to static heights if specified_heights is not None: From 61aa7df5b7e0d64819bb43a4d2402951a517589d Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Mar 2021 13:03:42 -0700 Subject: [PATCH 031/145] Fix logical statement --- mmctools/wrf/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 86f1675..fb756ea 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -868,7 +868,7 @@ def extract_column_from_wrfdata(fpath, coords, continue # 4D field specific processing - if field is 'T': + if field == 'T': # Add T0, set surface plane to TSK WRFdata[field] += T0 WRFdata[field] = add_surface_plane(WRFdata[field],plane=WRFdata['TSK']) From e4835db1a111fa249f8fa2a284653a2482c4ae70 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Mar 2021 13:46:15 -0700 Subject: [PATCH 032/145] Add `use_dimension_coords` option to wrfout_seriesReader Facilitates and expedites xarray operations on resulting dataset. False by default for backwards compatibility --- mmctools/wrf/utils.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index fb756ea..e64ea22 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1140,7 +1140,8 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, - hlim_ind=None,temp_var='THM'): + hlim_ind=None,temp_var='THM', + use_dimension_coords=False): """ Construct an a2e-mmc standard, xarrays-based, data structure from a series of 3-dimensional WRF output files @@ -1169,15 +1170,22 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, temp_var : str, optional Name of moist potential temperature variable, e.g., 'THM' for standard WRF output or 'T' MMC auxiliary output + use_dimension_coords : bool, optional + If True, then x and y coordinates will match the corresponding + dimension to facilitate and expedite xarray operations """ import wrf as wrfpy TH0 = 300.0 #WRF convention base-state theta = 300.0 K dims_dict = { 'Time':'datetime', 'bottom_top':'nz', - 'south_north': 'ny', - 'west_east':'nx', } + if use_dimension_coords: + dims_dict['west_east'] = 'x' + dims_dict['south_north'] = 'y' + else: + dims_dict['west_east'] = 'nx' + dims_dict['south_north'] = 'ny' ds = xr.open_mfdataset(os.path.join(wrf_path,wrf_file_filter), chunks={'Time': 10}, @@ -1237,7 +1245,7 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, ds_subset['wdir'] = xr.DataArray(180. + np.arctan2(ds_subset['u'],ds_subset['v'])*180./np.pi, dims=dim_keys) - # assign rename coord variable for time, and assign ccordinates + # rename coord variable for time and assign ccordinates ds_subset = ds_subset.rename({'XTIME': 'datetime'}) #Rename after defining the component DataArrays in the DataSet if specified_heights is None: ds_subset = ds_subset.assign_coords(z=ds_subset['z']) @@ -1245,9 +1253,12 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, ds_subset = ds_subset.assign_coords(x=ds_subset['x']) ds_subset = ds_subset.assign_coords(zsurface=ds_subset['zsurface']) ds_subset = ds_subset.rename_vars({'XLAT':'lat', 'XLONG':'lon'}) - #print(ds_subset) + for olddim,newdim in dims_dict.copy().items(): + if newdim in ds_subset.coords: + # have to swap dim instead of renaming if it already exists + ds_subset = ds_subset.swap_dims({olddim: newdim}) + dims_dict.pop(olddim) ds_subset = ds_subset.rename_dims(dims_dict) - #print(ds_subset) # Change by WHL to eliminate vertical info far from the surface to prevent memory crash try: From dd7bf77228ded937750bc4b01488fe87eb5344bb Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 2 Mar 2021 13:58:26 -0700 Subject: [PATCH 033/145] Adding check for ERA-Interim end of life ERA-Interim is not being made after 2019-09-10 12:00:00, so a check is added to make sure the dates that are requested are before this date. --- mmctools/wrf/preprocessing.py | 49 +++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 7871b51..a18ef65 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -141,7 +141,7 @@ def download(self,datetimes,path=None): class ERAInterim(RDADataset): """ERA-Interim Reanalysis - Note: Production stopped on August 31, 2019 + Note: Production stopped on September 10, 2019 Description: https://rda.ucar.edu/datasets/ds627.0/ """ @@ -162,23 +162,38 @@ def download(self,datetimes,path=None): path : str, optional Path to directory in which to save grib files """ - if path is None: - path = '.' + era_interim_end_date = '2019-09-10 12:00:00' + dates_before_end_of_era = True + good_dates = datetimes.copy() + for dt in datetimes: + if str(dt) > era_interim_end_date: + print('Bad date ({}) - after ERA-Interim EOL ({})'.format(str(dt),era_interim_end_date)) + good_dates = good_dates.drop(dt) + if len(good_dates) > 0: + datetimes = good_dates else: - os.makedirs(path,exist_ok=True) - # pressure-level data - - super().download(urlpath='ds627.0/{prefix:s}/%Y%m/{prefix:s}.{field:s}.%Y%m%d%H', - path=path, - prefix='ei.oper.an.pl', - datetimes=datetimes, - fields=['regn128sc','regn128uv']) - # surface data - super().download(urlpath='ds627.0/{prefix:s}/%Y%m/{prefix:s}.{field:s}.%Y%m%d%H', - path=path, - prefix='ei.oper.an.sfc', - datetimes=datetimes, - fields=['regn128sc']) + dates_before_end_of_era = False + print('WARNING: All dates are after ERA-Interim EOL ({})'.format(era_interim_end_date)) + print('Not downloading anything... Need to change reanalysis for these dates!') + + if dates_before_end_of_era: + if path is None: + path = '.' + else: + os.makedirs(path,exist_ok=True) + # pressure-level data + + super().download(urlpath='ds627.0/{prefix:s}/%Y%m/{prefix:s}.{field:s}.%Y%m%d%H', + path=path, + prefix='ei.oper.an.pl', + datetimes=datetimes, + fields=['regn128sc','regn128uv']) + # surface data + super().download(urlpath='ds627.0/{prefix:s}/%Y%m/{prefix:s}.{field:s}.%Y%m%d%H', + path=path, + prefix='ei.oper.an.sfc', + datetimes=datetimes, + fields=['regn128sc']) class MERRA2(RDADataset): From b15c93418cd2e97392f0c565dbd86c93602f6043 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Mar 2021 14:04:21 -0700 Subject: [PATCH 034/145] Cleanup vertical clipping code in wrfout_seriesReader() - move to before quantities of interest are calculated (reduce computations) - remove unnecessary try/except block to reflect intention indicated in the comments --- mmctools/wrf/utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index e64ea22..cb4d627 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1225,6 +1225,10 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, ds_subset['p'] = xr.DataArray(ds['P']+ds['PB'], dims=dim_keys) ds_subset['theta'] = xr.DataArray(ds[temp_var]+TH0, dims=dim_keys) + # clip vertical extent if requested + if hlim_ind is not None: + ds_subset = ds_subset.isel(bottom_top=slice(0, hlim_ind)) + # optionally, interpolate to static heights if specified_heights is not None: zarr = ds_subset['z'] @@ -1260,13 +1264,7 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, dims_dict.pop(olddim) ds_subset = ds_subset.rename_dims(dims_dict) - # Change by WHL to eliminate vertical info far from the surface to prevent memory crash - try: - ds_subset2 = ds_subset.isel( nz = slice(0, hlim_ind) ) - return ds_subset2 - except: - # if hlim_ind = None, default code execution - return ds_subset + return ds_subset def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): From 726790fdde5b815aee802114f5666944ef0d2c03 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 2 Mar 2021 14:04:36 -0700 Subject: [PATCH 035/145] Make "WARNING" more visible. --- mmctools/wrf/preprocessing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index a18ef65..7610cc9 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -167,7 +167,7 @@ def download(self,datetimes,path=None): good_dates = datetimes.copy() for dt in datetimes: if str(dt) > era_interim_end_date: - print('Bad date ({}) - after ERA-Interim EOL ({})'.format(str(dt),era_interim_end_date)) + print('WARNING: Bad date ({}) - after ERA-Interim EOL ({})'.format(str(dt),era_interim_end_date)) good_dates = good_dates.drop(dt) if len(good_dates) > 0: datetimes = good_dates @@ -861,7 +861,7 @@ def get_icbcs(self): optional_args['bounds'] = bounds else: print('We currently do not support ',icbc_type) - + icbc.download(datetimes,path=self.icbc_dir, **optional_args) From bd5e028f5c54727878d94dcba1c565aeb08f79ac Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Mar 2021 15:36:37 -0700 Subject: [PATCH 036/145] Add extra_vars kwarg to wrfout_seriesReader() Extra selected fields will be destaggered and interpolated (in z) if needed --- mmctools/wrf/utils.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index cb4d627..ecedbd7 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1141,6 +1141,7 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, hlim_ind=None,temp_var='THM', + extra_vars=[], use_dimension_coords=False): """ Construct an a2e-mmc standard, xarrays-based, data structure from a @@ -1167,6 +1168,8 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, memory error where the specified_heights argument is not well suited (i.e., want a range of non-interpolated heights), and you only care about data that are below a certain vertical index. + extra_vars : list, optional + List of additional fields to output temp_var : str, optional Name of moist potential temperature variable, e.g., 'THM' for standard WRF output or 'T' MMC auxiliary output @@ -1225,6 +1228,22 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, ds_subset['p'] = xr.DataArray(ds['P']+ds['PB'], dims=dim_keys) ds_subset['theta'] = xr.DataArray(ds[temp_var]+TH0, dims=dim_keys) + # extract additional variables if requested + for var in extra_vars: + if var not in ds.data_vars: + print(f'Requested variable "{var}" not in {str(list(ds.data_vars))}') + continue + field = ds[var] + newdims = list(field.dims) + print(f'Extracting {var} with dims {str(newdims)}...') + for idim, dim in enumerate(field.dims): + if dim.endswith('_stag'): + newdim = dim[:-len('_stag')] + print(f' destaggering {var} in {newdim}...') + field = wrfpy.destagger(field,stagger_dim=idim,meta=False) + newdims[idim] = newdim + ds_subset[var] = xr.DataArray(field, dims=newdims) + # clip vertical extent if requested if hlim_ind is not None: ds_subset = ds_subset.isel(bottom_top=slice(0, hlim_ind)) @@ -1232,7 +1251,9 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, # optionally, interpolate to static heights if specified_heights is not None: zarr = ds_subset['z'] - for var in ['u','v','w','p','theta']: + for var in ds_subset.data_vars: + if (var == 'z') or ('bottom_top' not in ds_subset[var].dims): + continue print('Interpolating',var) interpolated = wrfpy.interplevel(ds_subset[var], zarr, specified_heights) ds_subset[var] = interpolated #.expand_dims('Time', axis=0) From 67bb4a31f8a63f5c923ddaf1437de52e090675ae Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 2 Mar 2021 16:15:53 -0700 Subject: [PATCH 037/145] Add irange/jrange kwargs to wrfout_seriesReader() for subsetting --- mmctools/wrf/utils.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index ecedbd7..d68b55a 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1140,8 +1140,8 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, - hlim_ind=None,temp_var='THM', - extra_vars=[], + irange=None,jrange=None,hlim_ind=None, + temp_var='THM',extra_vars=[], use_dimension_coords=False): """ Construct an a2e-mmc standard, xarrays-based, data structure from a @@ -1161,7 +1161,12 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, If not None, then a list of static heights to which all data variables should be interpolated. Note that this significantly increases the data read time. - hlim_ind : int, index + irange,jrange : tuple, optional + If not none, then the DataArray ds_subset is further subset in + the horizontal dimensions, which should speed up execution. The + tuple should be (idxmin, idxmax), inclusive and 1-based indices + (as in WRF). + hlim_ind : int, index, optional If not none, then the DataArray ds_subset is further subset by vertical dimension, keeping vertical layers 0:hlim_ind. This is meant to be used to speed up execution of the code or prevent a @@ -1244,6 +1249,15 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, newdims[idim] = newdim ds_subset[var] = xr.DataArray(field, dims=newdims) + # subset in horizontal dimensions + # note: specified ranges are WRF indices, i.e., python indices +1 + if irange is not None: + assert isinstance(irange, tuple), 'irange should be (imin,imax)' + ds_subset = ds_subset.isel(west_east=slice(irange[0]-1, irange[1])) + if jrange is not None: + assert isinstance(jrange, tuple), 'jrange should be (jmin,jmax)' + ds_subset = ds_subset.isel(south_north=slice(jrange[0]-1, jrange[1])) + # clip vertical extent if requested if hlim_ind is not None: ds_subset = ds_subset.isel(bottom_top=slice(0, hlim_ind)) From f56c4ca515eeaa8b737b6ddccf4e08ae8fe74724 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 3 Mar 2021 10:58:32 -0700 Subject: [PATCH 038/145] Replace call to wrf.interplevel() with wrf.interpz3d() `interpz3d()` is apparently a lower-level routine and does not produce NaNs. Results from `interpz3d()` were verified by comparing against a manually calculated array from looping over Time/south_north/west_east and calling np.interp() for each column--the two arrays were identical. `interpz3d` also appears to have an added benefit of being more computationally efficient than `interplevel`; jupyter %timeit showed that the average compute time decreased by about 5%. --- mmctools/wrf/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index d68b55a..874246a 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1269,9 +1269,7 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, if (var == 'z') or ('bottom_top' not in ds_subset[var].dims): continue print('Interpolating',var) - interpolated = wrfpy.interplevel(ds_subset[var], zarr, specified_heights) - ds_subset[var] = interpolated #.expand_dims('Time', axis=0) - #print(ds_subset[var]) + ds_subset[var] = wrfpy.interpz3d(ds_subset[var], zarr, specified_heights) ds_subset = ds_subset.drop_dims('bottom_top').rename({'level':'z'}) dim_keys[1] = 'z' dims_dict.pop('bottom_top') From f6d8799aecc0e76101ceb0d13fda725e811bf8bf Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 3 Mar 2021 11:31:59 -0700 Subject: [PATCH 039/145] Code cleanup in wrfout_seriesReader() Instead of throwing out metadata and then recreating each DataArray in the output dataset, call wrf python functions with `meta=True` to return a DataArray with original metadata intact. I would expect this to help prevent "gotchas" by eliminating the extra step of manually setting the dimensions on the output arrays. --- mmctools/wrf/utils.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 874246a..6f570c8 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1205,14 +1205,14 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, ds_subset = ds[['XTIME']] print('Establishing coordinate variables, x,y,z, zSurface...') - zcoord = wrfpy.destagger((ds['PHB'] + ds['PH']) / 9.8, stagger_dim=1, meta=False) + ds_subset['z'] = wrfpy.destagger((ds['PHB'] + ds['PH']) / 9.8, + stagger_dim=1, meta=True) #ycoord = ds.DY * np.tile(0.5 + np.arange(ds.dims['south_north']), # (ds.dims['west_east'],1)) #xcoord = ds.DX * np.tile(0.5 + np.arange(ds.dims['west_east']), # (ds.dims['south_north'],1)) ycoord = ds.DY * (0.5 + np.arange(ds.dims['south_north'])) xcoord = ds.DX * (0.5 + np.arange(ds.dims['west_east'])) - ds_subset['z'] = xr.DataArray(zcoord, dims=dim_keys) #ds_subset['y'] = xr.DataArray(np.transpose(ycoord), dims=horiz_dim_keys) #ds_subset['x'] = xr.DataArray(xcoord, dims=horiz_dim_keys) ds_subset['y'] = xr.DataArray(ycoord, dims='south_north') @@ -1222,12 +1222,9 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, # for it to be time-varying for moving grids ds_subset['zsurface'] = xr.DataArray(ds['HGT'].isel(Time=0), dims=horiz_dim_keys) print('Destaggering data variables, u,v,w...') - ds_subset['u'] = xr.DataArray(wrfpy.destagger(ds['U'],stagger_dim=3,meta=False), - dims=dim_keys) - ds_subset['v'] = xr.DataArray(wrfpy.destagger(ds['V'],stagger_dim=2,meta=False), - dims=dim_keys) - ds_subset['w'] = xr.DataArray(wrfpy.destagger(ds['W'],stagger_dim=1,meta=False), - dims=dim_keys) + ds_subset['u'] = wrfpy.destagger(ds['U'], stagger_dim=3, meta=True) + ds_subset['v'] = wrfpy.destagger(ds['V'], stagger_dim=2, meta=True) + ds_subset['w'] = wrfpy.destagger(ds['W'], stagger_dim=1, meta=True) print('Extracting data variables, p,theta...') ds_subset['p'] = xr.DataArray(ds['P']+ds['PB'], dims=dim_keys) @@ -1239,15 +1236,12 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, print(f'Requested variable "{var}" not in {str(list(ds.data_vars))}') continue field = ds[var] - newdims = list(field.dims) - print(f'Extracting {var} with dims {str(newdims)}...') + print(f'Extracting {var}...') for idim, dim in enumerate(field.dims): if dim.endswith('_stag'): - newdim = dim[:-len('_stag')] - print(f' destaggering {var} in {newdim}...') - field = wrfpy.destagger(field,stagger_dim=idim,meta=False) - newdims[idim] = newdim - ds_subset[var] = xr.DataArray(field, dims=newdims) + print(f' destaggering {var} in dim {dim}...') + field = wrfpy.destagger(field, stagger_dim=idim, meta=True) + ds_subset[var] = field # subset in horizontal dimensions # note: specified ranges are WRF indices, i.e., python indices +1 @@ -1286,9 +1280,9 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, ds_subset = ds_subset.rename({'XTIME': 'datetime'}) #Rename after defining the component DataArrays in the DataSet if specified_heights is None: ds_subset = ds_subset.assign_coords(z=ds_subset['z']) - ds_subset = ds_subset.assign_coords(y=ds_subset['y']) - ds_subset = ds_subset.assign_coords(x=ds_subset['x']) - ds_subset = ds_subset.assign_coords(zsurface=ds_subset['zsurface']) + ds_subset = ds_subset.assign_coords(x=ds_subset['x'], + y=ds_subset['y'], + zsurface=ds_subset['zsurface']) ds_subset = ds_subset.rename_vars({'XLAT':'lat', 'XLONG':'lon'}) for olddim,newdim in dims_dict.copy().items(): if newdim in ds_subset.coords: From 01be13767772ccfa2595254cd027e8aaa6bb36f4 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 3 Mar 2021 14:29:27 -0700 Subject: [PATCH 040/145] Bug fix for dx,dy and add MERRA2 to IC/BCs dx and dy were being calculated incorrectly from the given dx,dy and the parent_grid_ratio. This has been fixed using np.prod() so that the correct dx,dy are calculated. MERRA2 currently has to be downloaded manually... adjusted the setup so that it doesn't download anything, links the manually downloaded files (need to be in icbc_directory/) in the submit_wps script, and does not run ungrib. --- mmctools/wrf/preprocessing.py | 37 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 7610cc9..2b25ee1 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -433,7 +433,6 @@ def _get_icbc_info(self): met_lvls = 30 elif icbc_type == 'MERRA2': met_lvls = 73 - download_freq = '1d' icbc_dict = {'type' : icbc_type, 'interval_seconds' : interval_seconds, @@ -659,7 +658,7 @@ def write_namelist_input(self): else: pid = pp parent_ids += '{0:>5},'.format(str(pid)) - dx_str += '{0:>5},'.format(str(int(self.namelist_opts['dxy']/pgr))) + dx_str += '{0:>5},'.format(str(int(self.namelist_opts['dxy']/np.prod(self.namelist_opts['parent_grid_ratio'][:(pp+1)])))) radt = self.namelist_opts['radt']/pgr if radt < 1: radt = 1 radt_str += '{0:>5},'.format(str(int(radt))) @@ -843,12 +842,14 @@ def get_icbcs(self): elif icbc_type == 'FNL': icbc = FNL() elif icbc_type == 'MERRA2': - icbc = MERRA2() - if 'resolution_deg' not in self.setup_keys(): - res_drag = 0 - else: - res_drag = self.setup_dict['resolution_deg'] - optional_args['resolution_deg'] = res_drag + print('Cannot download MERRA2 yet... please download manually and put in the IC/BC dir:') + print(self.icbc_dir) + #icbc = MERRA2() + #if 'resolution_deg' not in self.setup_keys(): + # res_drag = 0 + #else: + # res_drag = self.setup_dict['resolution_deg'] + #optional_args['resolution_deg'] = res_drag elif icbc_type == 'ERA5': icbc = ERA5() if 'bounds' not in self.setup_dict.keys(): @@ -861,8 +862,8 @@ def get_icbcs(self): optional_args['bounds'] = bounds else: print('We currently do not support ',icbc_type) - - icbc.download(datetimes,path=self.icbc_dir, **optional_args) + if icbc_type != 'MERRA2': + icbc.download(datetimes,path=self.icbc_dir, **optional_args) def write_submission_scripts(self,submission_dict,hpc='cheyenne'): @@ -900,18 +901,22 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): icbc_head = 'fnl_*' icbc_vtable = 'GFS' elif icbc_type == 'MERRA2': - icbc_head = 'MERRA2_*' + icbc_head = 'MERRA2*' icbc_vtable = 'GFS' else: print('We do not support this ICBC yet...') - icbc_files = '{}{}'.format(self.icbc_dir,icbc_head) - f.write("./link_grib.csh {}\n".format(icbc_files)) - f.write("ln -sf ungrib/Variable_Tables/Vtable.{} Vtable\n".format(icbc_vtable)) f.write("./geogrid.exe\n".format(executable)) - f.write("./ungrib.exe\n".format(executable)) + icbc_files = '{}{}'.format(self.icbc_dir,icbc_head) + if icbc_type == 'MERRA2': + f.write('ln -sf {} .\n'.format(icbc_files)) + else: + f.write("ln -sf ungrib/Variable_Tables/Vtable.{} Vtable\n".format(icbc_vtable)) + f.write("./link_grib.csh {}\n".format(icbc_files)) + f.write("./ungrib.exe\n".format(executable)) f.write("./metgrid.exe\n".format(executable)) - f.write("for i in GRIBFILE.*; do unlink $i; done\n") + if icbc_type != 'MERRA2': + f.write("for i in GRIBFILE.*; do unlink $i; done\n") else: f.write("mpiexec_mpt ./{}.exe\n".format(executable)) f.write("date_end=`date`\n") From ad92fb7a886c65163f10bd868dce431750409611 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 3 Mar 2021 16:29:20 -0700 Subject: [PATCH 041/145] Add agl option to wrfout_seriesReader() Revert to wrf.interplevels() as this is the interpolation routine found in the documentation Also, perform NaN check and print out warning if found --- mmctools/wrf/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 6f570c8..09a91f8 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1139,7 +1139,8 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest return dsF -def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, +def wrfout_seriesReader(wrf_path,wrf_file_filter, + specified_heights=None,agl=False, irange=None,jrange=None,hlim_ind=None, temp_var='THM',extra_vars=[], use_dimension_coords=False): @@ -1161,6 +1162,10 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, If not None, then a list of static heights to which all data variables should be interpolated. Note that this significantly increases the data read time. + agl : bool, optional + If True, then specified heights are expected to be above ground + level (AGL) and the interpolation heights will have the local + elevation subtracted out. irange,jrange : tuple, optional If not none, then the DataArray ds_subset is further subset in the horizontal dimensions, which should speed up execution. The @@ -1259,11 +1264,15 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter,specified_heights=None, # optionally, interpolate to static heights if specified_heights is not None: zarr = ds_subset['z'] + if agl: + zarr -= ds_subset['zsurface'] for var in ds_subset.data_vars: if (var == 'z') or ('bottom_top' not in ds_subset[var].dims): continue print('Interpolating',var) - ds_subset[var] = wrfpy.interpz3d(ds_subset[var], zarr, specified_heights) + ds_subset[var] = wrfpy.interplevel(ds_subset[var], zarr, specified_heights) + if np.any(~np.isfinite(ds_subset[var])): + print('WARNING: wrf.interplevel() produced NaNs -- make sure requested heights are in range and/or use agl=True') ds_subset = ds_subset.drop_dims('bottom_top').rename({'level':'z'}) dim_keys[1] = 'z' dims_dict.pop('bottom_top') From 3b2671831d2d6ac96cc175be1e394b679878bf1f Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 4 Mar 2021 14:59:09 -0700 Subject: [PATCH 042/145] Adding auxhist capability This will work for 1 or many auxhists. May add checks for variables at some point, but for now it requires all 4 (name, interval, frames per file, and file type). --- mmctools/wrf/preprocessing.py | 40 ++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 2b25ee1..30e5ecf 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -676,6 +676,37 @@ def write_namelist_input(self): io_str = self._get_nl_str(num_doms,self.namelist_opts['iofields_filename']) else: include_io = False + if 'auxinput4_inname' in self.namelist_opts.keys(): + include_aux4 = True + aux4_str_name = self.namelist_opts['auxinput4_inname'] + aux4_str_int = self._get_nl_str(num_doms,self.namelist_opts['auxinput4_interval']) + if 'io_form_auxinput4' in self.namelist_opts.keys(): + aux4_str_form = self.namelist_opts['io_form_auxinput4'] + else: + aux4_str_form = '2' + else: + include_aux4 = False + + aux_list = [] + for key in self.namelist_opts.keys(): + if ('auxhist' in key) and ('outname' in key): + aux_list.append(key.replace('auxhist','').replace('_outname','')) + include_auxout = False + if aux_list != []: + include_auxout = True + n_aux = len(aux_list) + aux_block = '' + for aa,aux in enumerate(aux_list): + aux_out_str = '{},'.format(self.namelist_opts['auxhist{}_outname'.format(aux)]) + aux_int_str = self._get_nl_str(num_doms,self.namelist_opts['auxhist{}_interval'.format(aux)]) + aux_per_str = self._get_nl_str(num_doms,self.namelist_opts['frames_per_auxhist{}'.format(aux)]) + aux_frm_str = '{},'.format(self.namelist_opts['io_form_auxhist{}'.format(aux)]) + line1 = ' auxhist{}_outname = {}\n'.format(aux,aux_out_str) + line2 = ' auxhist{}_interval = {}\n'.format(aux,aux_int_str) + line3 = ' frames_per_auxhist{} = {}\n'.format(aux,aux_per_str) + line4 = ' io_form_auxhist{} = {}\n'.format(aux,aux_frm_str) + aux_block += line1 + line2 + line3 + line4 + mp_str = self._get_nl_str(num_doms,self.namelist_opts['mp_physics']) sfclay_str = self._get_nl_str(num_doms,self.namelist_opts['sf_sfclay_physics']) surface_str = self._get_nl_str(num_doms,self.namelist_opts['sf_surface_physics']) @@ -707,6 +738,7 @@ def write_namelist_input(self): f = open('{}namelist.input'.format(self.run_dir),'w') f.write("&time_control\n") f.write(" run_days = 0,\n") + f.write(" run_hours = {0:>5},\n".format(run_hours)) f.write(" run_minutes = 0,\n") f.write(" run_seconds = 0,\n") @@ -735,6 +767,12 @@ def write_namelist_input(self): if include_io: f.write(" iofields_filename = {}\n".format(io_str)) f.write(" ignore_iofields_warning = .true.,\n") + if include_aux4: + f.write(" auxinput4_inname = {},\n".format(aux4_str_name)) + f.write(" auxinput4_interval = {}\n".format(aux4_str_int)) + f.write(" io_form_auxinput4 = {},\n".format(aux4_str_form)) + if include_auxout: + f.write(aux_block) f.write(" debug_level = {} \n".format(self.namelist_opts['debug'])) f.write("/\n") f.write("\n") @@ -1085,4 +1123,4 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a '{0:8.7f}'.format(float(twr_locx[tt]))) f.write(twr_line) f.close() - \ No newline at end of file + From 5bc632a528d2cfddad61c79561f623b99c16e49d Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 9 Mar 2021 14:10:29 -0700 Subject: [PATCH 043/145] Adding SST Overwrite class. This class will overwrite SST data in the met_em files with auxiliary SST data. Adding new datasets is not entirely trivial as many seem to have small quirks, but the majority of it is based in defining a new key in the sst_dict. --- mmctools/wrf/preprocessing.py | 340 +++++++++++++++++++++++++++++++++- 1 file changed, 339 insertions(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 30e5ecf..cb14048 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1123,4 +1123,342 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a '{0:8.7f}'.format(float(twr_locx[tt]))) f.write(twr_line) f.close() - + + + +sst_dict = { + 'OSTIA' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 5.5, # km + }, + + 'MUR' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 1.1, + }, + + 'MODIS' : { + 'time_dim' : 'time', + 'lat_dim' : 'latitude', + 'lon_dim' : 'longitude', + 'sst_name' : 'sst_data', + 'sst_dx' : 4.625, + }, + + 'GOES16' : { + 'time_dim' : 'time', + 'lat_dim' : 'lats', + 'lon_dim' : 'lons', + 'sst_name' : 'sea_surface_temperature', + 'sst_dx' : 2.0, + }, +} + +icbc_dict = { + 'ERAI' : { + 'sst_name' : 'SST', + }, + + 'FNL' : { + 'sst_name' : 'SKINTEMP', + }, + + 'ERA5' : { + 'sst_name' : 'SST', + }, +} + + +class overwrite_sst(): + ''' + Given WRF met_em files and auxiliary SST data, overwrite the SST data with + the new SST data. + + Inputs: + met_type = string; Initial / boundary conditions used (currently only ERAI, + ERA5, and FNL are supported) + overwrite_type = string; Name of SST data (OSTIA, MUT, MODIS) or FILL (replace + only missing values with SKINTEMP), or TSKIN (replace + all SST values with SKINTEMP) + met_directory = string; location of met_em files + sst_directory = string; location of sst files + out_directory = string; location to save new met_em files + smooth_opt = boolean; use smoothing over new SST data + fill_missing = boolean; fill missing values in SST data with SKINTEMP + + ''' + + def __init__(self, + met_type, + overwrite_type, + met_directory, + sst_directory, + out_directory, + smooth_opt=False, + fill_missing=False): + + self.met_type = met_type + self.overwrite = overwrite_type + self.met_dir = met_directory + self.sst_dir = sst_directory + self.out_dir = out_directory + self.smooth_opt = smooth_opt + self.fill_opt = fill_missing + + if overwrite_type == 'FILL': fill_missing=True + + if smooth_opt: + self.smooth_str = 'smooth' + else: + self.smooth_str = 'raw' + + self.out_dir += '{}/'.format(self.smooth_str) + + # Get met_em_files + self.met_em_files = sorted(glob.glob('{}met_em.d0*'.format(self.met_dir))) + + # Get SST data info (if not doing fill or tskin) + if (overwrite_type.upper() != 'FILL') and (overwrite_type.upper() != 'TSKIN'): + self._get_sst_info() + + # Overwrite the met_em SST data: + for mm,met_file in enumerate(self.met_em_files[::10]): + self._get_new_sst(met_file) + # If filling missing values with SKINTEMP: + if fill_missing: + self._fill_missing(met_file) + # Write to new file: + self._write_new_file(met_file) + + + def _get_sst_info(self): + self.sst_files = sorted(glob.glob('{}*.nc'.format(self.sst_dir))) + num_sst_files = len(self.sst_files) + sst_file_times = {} + for ff,fname in enumerate(self.sst_files): + sst = xr.open_dataset(fname) + + if ff == 0: + self.sst_lat = sst[sst_dict[self.overwrite]['lat_dim']] + self.sst_lon = sst[sst_dict[self.overwrite]['lon_dim']] + + if self.overwrite == 'MODIS': + f_time = [datetime.strptime(fname.split('/')[-1].split('.')[1],'%Y%m%d')] + else: + f_time = sst[sst_dict[self.overwrite]['time_dim']].data + + for ft in f_time: + ft = pd.to_datetime(ft) + sst_file_times[ft] = fname + self.sst_file_times = sst_file_times + + + def _get_new_sst(self,met_file): + met = xr.open_dataset(met_file) + + met_time = pd.to_datetime(met.Times.data[0].decode().replace('_',' ')) + met_lat = np.squeeze(met.XLAT_M) + met_lon = np.squeeze(met.XLONG_M) + met_landmask = np.squeeze(met.LANDMASK) + met_sst = np.squeeze(met[icbc_dict[self.met_type]['sst_name']]) + + if (self.overwrite.upper() != 'FILL') and (self.overwrite.upper() != 'TSKIN'): + + # Get window length for smoothing + if self.smooth_opt: + met_dx = met.DX/1000.0 + met_dy = met.DY/1000.0 + sst_dx = sst_dict[self.overwrite]['sst_dx'] + window = int(min([met_dx/sst_dx,met_dy/sst_dx])/2.0) + else: + window = 0 + + # Find closest SST files: + sst_neighbors = self._get_closest_files(met_time) + sst_weights = self._get_time_weights(met_time,sst_neighbors) + + before_ds = xr.open_dataset(self.sst_file_times[sst_neighbors[0]]) + after_ds = xr.open_dataset(self.sst_file_times[sst_neighbors[1]]) + + min_lon = np.max([np.nanmin(met_lon)-1,-180]) + max_lon = np.min([np.nanmax(met_lon)+1,180]) + + if (self.overwrite == 'MODIS') or (self.overwrite == 'GOES16'): + min_lat = np.min([np.nanmax(met_lat)+1,90]) + max_lat = np.max([np.nanmin(met_lat)-1,-90]) + else: + min_lat = np.max([np.nanmin(met_lat)-1,-90]) + max_lat = np.min([np.nanmax(met_lat)+1,90]) + + before_ds = before_ds.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), + sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) + after_ds = after_ds.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), + sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) + + # Select the time from the dataset: + if self.overwrite == 'MODIS': + # MODIS doesn't have time, so just squeeze: + before_sst = np.squeeze(before_ds[sst_dict[self.overwrite]['sst_name']]) + 273.15 + after_sst = np.squeeze(after_ds[sst_dict[self.overwrite]['sst_name']]) + 273.15 + else: + # Grab time and SST data: + before_sst = before_ds.sel({sst_dict[self.overwrite]['time_dim'] : sst_neighbors[0]})[sst_dict[self.overwrite]['sst_name']] + after_sst = after_ds.sel({sst_dict[self.overwrite]['time_dim'] : sst_neighbors[1]})[sst_dict[self.overwrite]['sst_name']] + if self.overwrite == 'GOES16': + before_sst += 273.15 + after_sst += 273.15 + + new_sst = met_sst.data.copy() + + sst_lat = self.sst_lat.data + sst_lon = self.sst_lon.data + + for jj in met.south_north: + for ii in met.west_east: + if met_landmask[jj,ii] == 0.0: + dist_lat = abs(sst_lat - float(met_lat[jj,ii])) + dist_lon = abs(sst_lon - float(met_lon[jj,ii])) + + lat_ind = np.where(dist_lat==np.min(dist_lat))[0] + lon_ind = np.where(dist_lon==np.min(dist_lon))[0] + + if (len(lat_ind) > 1) and (len(lon_ind) > 1): + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[1] + window] + lon_s = sst_lon[lon_ind[0] - window] + lon_e = sst_lon[lon_ind[1] + window] + + elif (len(lat_ind) > 1) and (len(lon_ind) == 1): + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[1] + window] + lon_s = sst_lon[lon_ind[0] - window] + lon_e = sst_lon[lon_ind[0] + window] + + elif (len(lat_ind) == 1) and (len(lon_ind) > 1): + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[0] + window] + lon_s = sst_lon[lon_ind[0] - window] + lon_e = sst_lon[lon_ind[1] + window] + + else: + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[0] + window] + lon_s = sst_lon[lon_ind[0] - window] + try: + lon_e = sst_lon[lon_ind[0] + window] + except IndexError: + lon_e = len(sst_lon) + + sst_before_val = before_sst.sel({ + sst_dict[self.overwrite]['lat_dim']:slice(lat_s,lat_e), + sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) + + sst_after_val = after_sst.sel({ + sst_dict[self.overwrite]['lat_dim']:slice(lat_s,lat_e), + sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) + + new_sst[jj,ii] = sst_before_val*sst_weights[0] + sst_after_val*sst_weights[1] + + else: + if (self.overwrite.upper() == 'TSKIN'): + new_sst = met_sst.data.copy() + tsk = np.squeeze(met.SKINTEMP).data + new_sst[np.where(met_landmask==0.0)] = tsk[np.where(met_landmask==0.0)] + elif (self.overwrite.upper() == 'FILL'): + new_sst = np.squeeze(met[icbc_dict[self.met_type]['sst_name']]).data + self.new_sst = new_sst + + + def _get_closest_files(self,met_time): + sst_times = np.asarray(list(self.sst_file_times.keys())) + + if (met_time > max(sst_times)) or (met_time < min(sst_times)): + raise Exception("met_em time out of range of SST times:\nSST min,max = {}, {}\nmet_time = {}".format( + min(sst_times),max(sst_times),met_time)) + + time_dist = sst_times.copy() + for dt,stime in enumerate(sst_times): + time_dist[dt] = abs(stime - met_time) + + closest_time = sst_times[np.where(time_dist == np.min(time_dist))] + + + if len(closest_time) == 1: + if closest_time == met_time: + sst_before = closest_time[0] + sst_after = closest_time[0] + else: + got_before = False + got_after = False + closest_time = closest_time[0] + if (closest_time - met_time).total_seconds() < 0: + sst_before = closest_time + got_before = True + else: + sst_after = closest_time + got_after = True + + next_closest_dist = list(time_dist) + next_closest_times = list(sst_times) + next_closest_dist.remove(np.min(time_dist)) + next_closest_times.remove(closest_time) + next_closest_dist = np.asarray(next_closest_dist) + next_closest_times = np.asarray(next_closest_times) + next_closest_time = next_closest_times[np.where(next_closest_dist == np.min(next_closest_dist))][0] + + if got_before: + sst_after = next_closest_time + if got_after: + sst_before = next_closest_time + + elif len(closest_time) == 2: + sst_before = closest_time[0] + sst_after = closest_time[1] + + return([sst_before,sst_after]) + + + def _get_time_weights(self,met_time,times): + before = times[0] + after = times[1] + + d1 = (met_time - before).seconds + d2 = (after - met_time).seconds + if d1 == 0 & d2 == 0: + w1,w2 = 1.0,0.0 + else: + w1 = d2/(d1+d2) + w2 = d1/(d1+d2) + + return([w1,w2]) + + + def _fill_missing(self,met_file): + met = xr.open_dataset(met_file) + tsk = np.squeeze(met.SKINTEMP) + met_landmask = np.squeeze(met.LANDMASK) + bad_inds = np.where(((met_landmask == 0) & (self.new_sst == 0.0))) + self.new_sst[bad_inds] = tsk.data[bad_inds] + bad_inds = np.where(np.isnan(self.new_sst)) + self.new_sst[bad_inds] = tsk.data[bad_inds] + + + def _write_new_file(self,met_file): + f_name = met_file.split('/')[-1] + new_file = self.out_dir + f_name + print(new_file) + new = xr.open_dataset(met_file) + new.SST.data = np.expand_dims(self.new_sst,axis=0) + new.attrs['source'] = '{}'.format(self.overwrite) + new.attrs['smoothed'] = '{}'.format(self.smooth_opt) + new.attrs['filled'] = '{}'.format(self.fill_opt) + if os.path.exists(new_file): + print('File exists... replacing') + os.remove(new_file) + new.to_netcdf(new_file) From 0784e30beca78b35e6e4a89fb925bf3524c688d6 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 9 Mar 2021 14:53:15 -0700 Subject: [PATCH 044/145] Bug fix for getting nearest times. Old way allowed for both times to be before/after as opposed to one before and one after. This makes sure that one is before and one is after so that linear interpolation works. --- mmctools/wrf/preprocessing.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index cb14048..495e0cb 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1388,7 +1388,6 @@ def _get_closest_files(self,met_time): closest_time = sst_times[np.where(time_dist == np.min(time_dist))] - if len(closest_time) == 1: if closest_time == met_time: sst_before = closest_time[0] @@ -1397,24 +1396,25 @@ def _get_closest_files(self,met_time): got_before = False got_after = False closest_time = closest_time[0] + closest_ind = int(np.where(sst_times == closest_time)[0]) if (closest_time - met_time).total_seconds() < 0: sst_before = closest_time + next_closest_times = sst_times[closest_ind+1:] + next_closest_dist = time_dist[closest_ind+1:] got_before = True else: sst_after = closest_time + next_closest_times = sst_times[:closest_ind-1] + next_closest_dist = time_dist[:closest_ind-1] got_after = True - next_closest_dist = list(time_dist) - next_closest_times = list(sst_times) - next_closest_dist.remove(np.min(time_dist)) - next_closest_times.remove(closest_time) - next_closest_dist = np.asarray(next_closest_dist) - next_closest_times = np.asarray(next_closest_times) next_closest_time = next_closest_times[np.where(next_closest_dist == np.min(next_closest_dist))][0] if got_before: + assert (next_closest_time - met_time).total_seconds() >= 0.0, 'Next closest time not after first time.' sst_after = next_closest_time if got_after: + assert (next_closest_time - met_time).total_seconds() <= 0.0, 'Next closest time not before first time.' sst_before = next_closest_time elif len(closest_time) == 2: From d3502e56ef06f2af5d8ec1d244fcb768190d1fcd Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 9 Mar 2021 15:38:07 -0700 Subject: [PATCH 045/145] Forgot to remove testing edit. --- mmctools/wrf/preprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 495e0cb..7431803 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1228,7 +1228,7 @@ def __init__(self, self._get_sst_info() # Overwrite the met_em SST data: - for mm,met_file in enumerate(self.met_em_files[::10]): + for mm,met_file in enumerate(self.met_em_files): self._get_new_sst(met_file) # If filling missing values with SKINTEMP: if fill_missing: From 8e083e3886b0217095b90551fe2d8c8c92ca6546 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 9 Mar 2021 15:49:45 -0700 Subject: [PATCH 046/145] Change class names to fit PEP-8 --- mmctools/wrf/preprocessing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 7431803..3e3f3b6 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -399,7 +399,7 @@ def download(self,datetimes,path=None,bounds={}): ) -class setup_wrf(): +class SetupWRF(): ''' Set up run directory for WRF / WPS ''' @@ -1175,7 +1175,7 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a } -class overwrite_sst(): +class OverwriteSST(): ''' Given WRF met_em files and auxiliary SST data, overwrite the SST data with the new SST data. From 27775909e85c45007ffd7b81c2e64be4a8aa2b91 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 9 Mar 2021 15:57:51 -0700 Subject: [PATCH 047/145] Xarray needed for OverwriteSST --- mmctools/wrf/preprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 3e3f3b6..4de1cae 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd import glob +import xarray as xr def prompt(s): if sys.version_info[0] < 3: From a0b4d38ba9d5cb22c7214d5c858f8b8e694ac186 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 11 Mar 2021 09:44:22 -0700 Subject: [PATCH 048/145] Making NaNs = 0.0 (as expected in WRF) --- mmctools/wrf/preprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 4de1cae..ec7f37c 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1234,6 +1234,7 @@ def __init__(self, # If filling missing values with SKINTEMP: if fill_missing: self._fill_missing(met_file) + self.new_sst = np.nan_to_num(self.new_sst) # Write to new file: self._write_new_file(met_file) From b788d5314fee9ae9c31825fa78d5d197df43f145 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 18 Mar 2021 16:20:52 -0600 Subject: [PATCH 049/145] Adding MERRA2 capability --- mmctools/helper_functions.py | 71 ++++++++++++++++++++++++++++++++++- mmctools/wrf/preprocessing.py | 23 ++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 347f2a2..1f5e8b4 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -5,7 +5,8 @@ import pandas as pd import xarray as xr import time - +import glob +from datetime import datetime,timedelta # constants epsilon = 0.622 # ratio of molecular weights of water to dry air @@ -272,6 +273,7 @@ def power_spectral_density(df,tstart=None,interval=None,window_size='10min', # Determine time scale timevalues = df.index.get_level_values(0) + if isinstance(timevalues,pd.DatetimeIndex): timescale = pd.to_timedelta(1,'s') else: @@ -293,6 +295,7 @@ def power_spectral_density(df,tstart=None,interval=None,window_size='10min', # Determine sampling rate and samples per window dts = np.diff(timevalues.unique())/timescale dt = dts[0] + if type(window_type) is str: nperseg = int( pd.to_timedelta(window_size)/pd.to_timedelta(dt,'s') ) else: @@ -1002,3 +1005,69 @@ def estimate_ABL_height(T=None,Tw=None,uw=None,sanitycheck=True,**kwargs): ablh.name = 'ABLheight' return ablh +def get_nc_file_times(f_dir, + f_grep_str, + decode_times=True, + time_dim='time', + get_time_from_fname=False, + f_split=[], + time_pos=[], + time_fmt='%Y%m%d'): + ''' + Get times from netCDF files and returns dictionary of times associated with the file that it's in: + dict{'time' : 'file_path'}. This uses xarray to find the times. + + This is useful for when you're using different NetCDF datasets that have non-uniform time + conventions (i.e., some have 1 time per file, others have multiple.) + + f_dir : str + path to files + f_grep_str : str + string to grep the file - should include '*' + decode_times : bool + (Default=True) If you want xarray to decode the times (if xarray cannot decode the time, set + this to False) + time_dim : str + (Default='time') time dimension name + get_time_from_fname : bool + if there is no time in the file, you can use the following options to parse the file name + f_split : list + The strings (in order) for which the file name should be parsed + time_pos : list (same dimension as f_split) + After the string has been split, which index should be taken. Must be same dimension and + order as f_split to work properly + time_fmt : str + (Default '%Y%m%d') Format for the datetime in file. + ''' + files = sorted(glob.glob('{}{}'.format(f_dir,f_grep_str))) + num_files = len(files) + file_times = {} + + for ff,fname in enumerate(files): + ncf = xr.open_dataset(fname,decode_times=decode_times) + + #ncf = ncdf(fname,'r') + + if get_time_from_fname: + assert f_split != [], 'Need to specify how to split the file name.' + assert time_pos != [], 'Need to specify index of time string after split.' + f_name = fname.replace(f_dir,'') + assert len(f_split) == len(time_pos), 'f_split (how to parse the file name) and time_pos (index of time string is after split) must be same size.' + for split,pos in zip(f_split,time_pos): + f_name = f_name.split(split)[pos] + + f_time = [datetime.strptime(f_name,time_fmt)] + else: + if not decode_times: + nc_times = ncf[time_dim][:].data + f_time = [] + for ff,nc_time in enumerate(nc_times): + time_start = pd.to_datetime(ncf[time_dim].units.replace('seconds since ','')) + f_time.append(datetime(time_start.year, time_start.month, time_start.day) + timedelta(seconds=int(nc_time))) + else: + f_time = ncf[time_dim].data + + for ft in f_time: + ft = pd.to_datetime(ft) + file_times[ft] = fname + return (file_times) \ No newline at end of file diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index ec7f37c..c1e8ebc 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -5,6 +5,7 @@ import pandas as pd import glob import xarray as xr +from mmctools.helper_functions import get_nc_file_times def prompt(s): if sys.version_info[0] < 3: @@ -1240,6 +1241,22 @@ def __init__(self, def _get_sst_info(self): + + if self.overwrite == 'MODIS': + get_time_from_fname = True + else: + get_time_from_fname = False + + sst_file_times = get_nc_file_times(f_dir='{}'.format(self.sst_dir), + f_grep_str='*.nc', + decode_times=True, + time_dim=sst_dict[self.overwrite]['time_dim'], + get_time_from_fname=get_time_from_fname, + f_split=['.'], + time_pos=[1]) + + + ''' self.sst_files = sorted(glob.glob('{}*.nc'.format(self.sst_dir))) num_sst_files = len(self.sst_files) sst_file_times = {} @@ -1258,7 +1275,13 @@ def _get_sst_info(self): for ft in f_time: ft = pd.to_datetime(ft) sst_file_times[ft] = fname + ''' self.sst_file_times = sst_file_times + + sst = xr.open_dataset(sst_file_times[list(sst_file_times.keys())[0]]) + self.sst_lat = sst[sst_dict[self.overwrite]['lat_dim']] + self.sst_lon = sst[sst_dict[self.overwrite]['lon_dim']] + def _get_new_sst(self,met_file): From ab8c0b9849034a4be2339cc1ec01a8809ee73472 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Fri, 19 Mar 2021 13:54:19 -0600 Subject: [PATCH 050/145] Adding calc_spectra() function. This function should be used instead of the model4D_*spectra functions. --- mmctools/helper_functions.py | 201 ++++++++++++++++++++++++++++++++++- 1 file changed, 197 insertions(+), 4 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 1f5e8b4..8d20681 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -259,7 +259,8 @@ def covariance(a,b,interval='10min',resample=False,**kwargs): def power_spectral_density(df,tstart=None,interval=None,window_size='10min', - window_type='hanning',detrend='linear',scaling='density'): + window_type='hanning',detrend='linear',scaling='density', + num_overlap=None): """ Calculate power spectral density using welch method and return a new dataframe. The spectrum is calculated for every column @@ -309,8 +310,9 @@ def power_spectral_density(df,tstart=None,interval=None,window_size='10min', spectra = {} for col in df.columns: - f,P = welch( df.loc[inrange,col], fs=1./dt, nperseg=nperseg, - detrend=detrend,window=window_type,scaling=scaling) + f,P = welch(df.loc[inrange,col], fs=1./dt, nperseg=nperseg, + detrend=detrend,window=window_type,scaling=scaling, + noverlap=num_overlap) spectra[col] = P spectra['frequency'] = f return pd.DataFrame(spectra).set_index('frequency') @@ -1070,4 +1072,195 @@ def get_nc_file_times(f_dir, for ft in f_time: ft = pd.to_datetime(ft) file_times[ft] = fname - return (file_times) \ No newline at end of file + return (file_times) + +def calc_spectra(data, + var_oi=None, + spectra_dim=None, + average_dim=None, + level_dim=None, + level=None, + window='hamming', + number_of_windows=1, + window_length=None, + window_overlap_pct=None, + detrend='constant' + ): + + ''' + Calculate spectra using the Welch function. This code uses the + power_spectral_density function from helper_functions.py. This function + accepts either xarray dataset or dataArray, or pandas dataframe. Dimensions + must be 4 or less (time, x, y, z). Returns a xarray dataset with the PSD of + the variable (f(average_dim, level, frequency/wavelength)) and the frequency + or wavelength variables. Averages of the PSD over time or space can easily + be done with xarray.Dataset.mean(dim='[dimension_name]'). + + Parameters + ========== + data : xr.Dataset, xr.DataArray, or pd.dataframe + The data that spectra should be calculated over + var_oi : str + Variable of interest - what variable should PSD be computed from. + spectra_dim : str + Name of the dimension that the variable spans for spectra to be + computed. E.g., if you want time spectra, this should be something like + 'time' or 'datetime', if you want spatial spectra, this should be 'x' or + 'y' (or for WRF, 'south_north' / 'west_east') + average_dim : str + Which dimension should be looped over for averaging. Name should be + similar to what is described in spectra_dim + level_dim : str (optional) + If you have a third dimension that you want to loop over, specify the + dimension name here. E.g., if you want to calculate PSD at several + heights, level_dim = 'height_dim' + level : list, array, int (optional) + If there is a level_dim, what levels should be looped over. Default is + the length of level_dim. + window : 'hamming' or specific window (optional) + What window should be used for the PSD calculation? If None, no window + is used in the Welch function (window is all 1's). + number_of_windows : int (optional) + Number of windows - determines window length as signal length / int + window_length : int or str (optional) + Alternative to number_of_windows, you can directly specify the length + of the windows as an integer or as a string to be converted to a + time_delta. This will overwrite number_of_windows. If using time_delta, + the window_length cannot be shorter than the data frequency. + overlap_percent : int (optional) + Percentage of data overlap with respect to window length. + detrend : str (optional) + Should the data be detrended (constant, linear, etc.). See Welch + function for more details. + + Example Call + ============ + + psd = calc_spectra(data, # data read in with xarray + var_oi='W', # PSD of 'W' to be computed + spectra_dim='west_east', # Take the west-east line + average_dim='south_north', # Average over north/south + level_dim='bottom_top_stag', # Compute over each level + level=None) # level defaults to all levels in array + + ''' + from scipy.signal.windows import hamming + + # Datasets, DataArrays, or dataframes + if not isinstance(data,xr.Dataset): + if isinstance(data,pd.DataFrame): + data = data.to_xarray() + elif isinstance(data,xr.DataArray): + if data.name is None: + data.name = var_oi + data = data.to_dataset() + else: + raise ValueError('unsupported type: {}'.format(type(data))) + + # Get index for frequency / wavelength: + spec_index = data.coords[spectra_dim] + dX = (spec_index.data[1] - spec_index.data[0]) + if isinstance(dX,(pd.Timedelta,np.timedelta64)): + dX = pd.to_timedelta(dX)#.total_seconds() + else: + dX = float(dX) + + # Window length specification: + if window_length is not None: + if (isinstance(window_length,str)): + if isinstance(dX,(pd.Timedelta,np.timedelta64)): + try: + dwindow = pd.to_timedelta(window_length) + except: + raise ValueError('Cannot convert {} to timedelta'.format(window_length)) + + if dwindow < dX: + raise ValueError('window_length is smaller than data time spacing') + nblock = int( dwindow/dX ) + else: + raise ValueError('window_length given as timedelta, but spectra_dim is not datetime...') + else: + nblock = int(window_length) + else: + nblock = int((len(data[spectra_dim].data))/number_of_windows) + + # Create window: + if window is None: + window = np.ones(nblock) + elif (window == 'hamming') or (window == 'hanning'): + window = hamming(nblock, True) #Assumed non-periodic in the spectra_dim + + # Calculate number of overlapping points: + if window_overlap_pct is not None: + if window_overlap_pct > 1: + window_overlap_pct /= 100.0 + num_overlap = int(nblock*window_overlap_pct) + else: + num_overlap = None + + # Make sure 'level' is iterable: + if level is None: + if level_dim is not None: + level = data[level_dim].data[:] + else: + level = [None] + elif isinstance(level,(int,float)): + level = [level] + level = list(level) + n_levels = len(level) + + if average_dim is None: + average_dim_data = [None] + else: + average_dim_data = data[average_dim] + + for ll,lvl in enumerate(level): + if lvl is not None: + spec_dat_lvl = data.sel({level_dim:lvl}) + else: + spec_dat_lvl = data.copy() + for ad,avg_dim in enumerate(average_dim_data): + if avg_dim is not None: + spec_dat = spec_dat_lvl.sel({average_dim:avg_dim}) + else: + spec_dat = spec_dat_lvl.copy() + if len(list(spec_dat.dims)) > 1: + dim_list = list(spec_dat.dims) + dim_list.remove(spectra_dim) + assert len(dim_list) == 1, 'There are too many dimensions... drop one of {}'.format(dim_list) + assert len(spec_dat[dim_list[0]].data) == 1, 'Not sure how to parse this dimension, {}, reduce to 1 or remove'.format(dim_list) + spec_dat = spec_dat.squeeze() + for varn in list(spec_dat.variables.keys()): + if (varn != var_oi) and (varn != spectra_dim): + spec_dat = spec_dat.drop(varn) + + spec_dat_df = spec_dat[var_oi].to_dataframe() + + psd = power_spectral_density(spec_dat_df, + window_type=window, + detrend=detrend, + num_overlap=num_overlap) + psd = psd.to_xarray() + if avg_dim is not None: + psd = psd.assign_coords({average_dim:1}) + psd[average_dim] = avg_dim.data + psd = psd.expand_dims(average_dim) + + if ad == 0: + psd_level = psd + else: + psd_level = psd.combine_first(psd_level) + else: + psd_level = psd + + if level_dim is not None: + psd_level = psd_level.assign_coords({level_dim:1}) + psd_level[level_dim] = lvl#.data + psd_level = psd_level.expand_dims(level_dim) + + if ll == 0: + psd_f = psd_level + else: + psd_f = psd_level.combine_first(psd_f) + return(psd_f) + From c68678b4883034e86b88ddcf5f4802d5c35b3195 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Mon, 22 Mar 2021 10:33:39 -0600 Subject: [PATCH 051/145] Adding more options to the namelist --- mmctools/wrf/preprocessing.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 30e5ecf..b9cff14 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -512,6 +512,7 @@ def _check_namelist_opts(self): 'relax_zone' : 4, 'nio_tasks_per_group' : 0, 'nio_groups' : 1, + 'sst_update' : 1, } namelist_opts = namelist_defaults @@ -825,12 +826,18 @@ def write_namelist_input(self): f.write(" num_soil_layers = {}, \n".format(self.namelist_opts['num_soil_layers'])) f.write(" num_land_cat = {}, \n".format(self.namelist_opts['num_land_cat'])) f.write(" sf_urban_physics = {}\n".format(urb_str)) + f.write(" sst_update = {}, \n".format(self.namelist_opts['sst_update'])) f.write(" /\n") f.write("\n") f.write("&fdda\n") f.write("/\n") f.write("\n") f.write("&dynamics\n") + if 'hybrid_opt' in self.namelist_opts: + f.write(" w_damping = {}, \n".format(self.namelist_opts['hybrid_opt'])) + if 'use_theta_m' in self.namelist_opts: + f.write(" use_theta_m = {}, \n".format(self.namelist_opts['use_theta_m'])) + f.write(" w_damping = {}, \n".format(self.namelist_opts['w_damping'])) f.write(" diff_opt = {}\n".format(diff_str)) f.write(" km_opt = {}\n".format(km_str)) @@ -990,6 +997,10 @@ def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): add_str_start = '+:h:0:' for ii,io_name in enumerate(np.unique(io_names)): + if '"' in io_name: + io_name = io_name.replace('"','') + if "'" in io_name: + io_name = io_name.replace("'",'') f = open('{}{}'.format(self.run_dir,io_name),'w') line = '' if vars_to_remove is not None: From 0cd0862a14b1a3e19ad90b1bc066fa1b3da28d86 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Mon, 22 Mar 2021 10:41:52 -0600 Subject: [PATCH 052/145] Adding tstart and interval for PSD calculation If somebody wants to only consider a portion of the data for PSD, this allows them to pick the start and the length of the data. --- mmctools/helper_functions.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 8d20681..1097fac 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1084,7 +1084,9 @@ def calc_spectra(data, number_of_windows=1, window_length=None, window_overlap_pct=None, - detrend='constant' + detrend='constant', + tstart=None, + interval=None ): ''' @@ -1132,6 +1134,14 @@ def calc_spectra(data, detrend : str (optional) Should the data be detrended (constant, linear, etc.). See Welch function for more details. + tstart : datetime (optional) + If calculating the spectra over only a portion of the data, when will + the series start (only available for timeseries at the moment). + interval : str (optional) + If calculating the spectra over only a portion of the data, how long + of a segment is considered (only available for timeseries at the + moment). + Example Call ============ @@ -1239,7 +1249,8 @@ def calc_spectra(data, psd = power_spectral_density(spec_dat_df, window_type=window, detrend=detrend, - num_overlap=num_overlap) + num_overlap=num_overlap, + tstart=tstart,interval=interval) psd = psd.to_xarray() if avg_dim is not None: psd = psd.assign_coords({average_dim:1}) From 54c4ad1aaec8e5581db29fad170709788184e627 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Mar 2021 08:20:55 -0600 Subject: [PATCH 053/145] Bug fix: tslist options mixed. Correctly setting the tslist options for number of locations and max levels. --- mmctools/wrf/preprocessing.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index b9cff14..2366ea4 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -471,8 +471,8 @@ def _check_namelist_opts(self): 'restart_interval' : 360, 'frames_per_file' : 1, 'debug' : 0, - 'ts_locations' : 20, - 'ts_levels' : self.setup_dict['num_eta_levels'], + 'max_ts_locs' : 20, + 'max_ts_level' : self.setup_dict['num_eta_levels'], 'mp_physics' : 10, 'ra_lw' : 4, 'ra_sw' : 4, @@ -538,10 +538,10 @@ def link_executables(self): os.makedirs(self.run_dir) # Link WPS and WRF files / executables - wrf_files = glob.glob('{}[!n]*'.format(self.wrf_exe_dir)) - self._link_files(wrf_files,self.run_dir) wps_files = glob.glob('{}[!n]*'.format(self.wps_exe_dir)) self._link_files(wps_files,self.run_dir) + wrf_files = glob.glob('{}[!n]*'.format(self.wrf_exe_dir)) + self._link_files(wrf_files,self.run_dir) def _get_nl_str(self,num_doms,phys_opt): phys_str = '' @@ -782,8 +782,8 @@ def write_namelist_input(self): f.write(" time_step_fract_num = {},\n".format(ts_num)) f.write(" time_step_fract_den = {},\n".format(ts_den)) f.write(" max_dom = {},\n".format(num_doms)) - f.write(" max_ts_locs = {},\n".format(self.namelist_opts['ts_locations'])) - f.write(" max_ts_level = {},\n".format(self.namelist_opts['ts_locations'])) + f.write(" max_ts_locs = {},\n".format(self.namelist_opts['max_ts_locs'])) + f.write(" max_ts_level = {},\n".format(self.namelist_opts['max_ts_level'])) f.write(" tslist_unstagger_winds = .true., \n") f.write(" s_we = {}\n".format("{0:>5},".format(1)*num_doms)) f.write(" e_we = {}\n".format(nx_str)) @@ -834,7 +834,7 @@ def write_namelist_input(self): f.write("\n") f.write("&dynamics\n") if 'hybrid_opt' in self.namelist_opts: - f.write(" w_damping = {}, \n".format(self.namelist_opts['hybrid_opt'])) + f.write(" hybrid_opt = {}, \n".format(self.namelist_opts['hybrid_opt'])) if 'use_theta_m' in self.namelist_opts: f.write(" use_theta_m = {}, \n".format(self.namelist_opts['use_theta_m'])) From 399d4595da26b26413e67214f1ebd255720a2643 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Mar 2021 08:23:31 -0600 Subject: [PATCH 054/145] Adding check to see if there are no tower files. This has tripped me up several times: if you mess up the path or the domain_of_interest you'll get some weird error down the line that makes it difficult to debug and turns out that the program just can't find tower files. Adding this check up front fixes the issue and will tell you something is wrong at the start. --- mmctools/wrf/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 3594ce4..c2f45dd 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1112,7 +1112,7 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest ntimes = np.shape(restarts)[0] floc = '{}{}/*{}.??'.format(fdir,restarts[0],domain_of_interest) file_list = glob.glob(floc) - + assert file_list != [], 'No tslist files found. Check kwargs.' for ff,file in enumerate(file_list): file = file[:-3] file_list[ff] = file From 1c64ad1e45191b4ad66d0ab9b561c23792272b3a Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 25 Mar 2021 11:39:55 -0600 Subject: [PATCH 055/145] Adding level selection nearest --- mmctools/helper_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 1097fac..8f0bedf 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1226,7 +1226,8 @@ def calc_spectra(data, for ll,lvl in enumerate(level): if lvl is not None: - spec_dat_lvl = data.sel({level_dim:lvl}) + spec_dat_lvl = data.sel({level_dim:lvl},method='nearest') + lvl = spec_dat_lvl[level_dim].data else: spec_dat_lvl = data.copy() for ad,avg_dim in enumerate(average_dim_data): From 5725ba5fb017158b6be59657355596bc98122a58 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Mon, 29 Mar 2021 14:45:18 -0600 Subject: [PATCH 056/145] Add functions for TRI and VRM Functions for Terrain Ruggedness Index (TRI) and Vector Ruggedness Measure (VRM) accept lists, arrays, and xr.Dataset and DataArray. Returned are arrays with the respective TRI or VRM values. Both of these calculations rely on a height array and a specified window. --- mmctools/helper_functions.py | 79 ++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 8f0bedf..c44163f 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1276,3 +1276,82 @@ def calc_spectra(data, psd_f = psd_level.combine_first(psd_f) return(psd_f) + +def calcTRI(hgt,window): + ''' + Terrain Ruggedness Index + Riley, S. J., DeGloria, S. D., & Elliot, R. (1999). Index that + quantifies topographic heterogeneity. intermountain Journal + of sciences, 5(1-4), 23-27. + + hgt : array + Array of heights over which TRI will be calculated + window : int + Length of window in x and y direction. Must be odd. + ''' + + # Window setup: + assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' + Hwindow = int(np.floor(window/2)) + + # Type and dimension check: + if isinstance(hgt,(xr.Dataset,xr.DataArray)): + hgt = hgt.data + assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) + + ny,nx = np.shape(hgt) + tri = np.zeros((ny,nx)) + # Loop over all cells within bounds of window: + for ii in range(Hwindow+1,nx-Hwindow-1): + for jj in range(Hwindow+1,ny-Hwindow-1): + hgt_window = hgt[jj-Hwindow:jj+Hwindow+1,ii-Hwindow:ii+Hwindow+1] + tri[jj,ii] = (np.sum((hgt_window - hgt[jj,ii])**2.0))**0.5 + return tri + + +def calcVRM(hgt,window): + ''' + Vector Ruggedness Measure + Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). + Quantifying landscape ruggedness for animal habitat analysis: + a case study using bighorn sheep in the Mojave Desert. The + Journal of wildlife management, 71(5), 1419-1426. + + hgt : array + Array of heights over which TRI will be calculated + window : int + Length of window in x and y direction. Must be odd. + ''' + import richdem as rd + + # Window setup: + assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' + Hwndw = int(np.floor(window/2)) + + # Type and dimension check: + if isinstance(hgt,(xr.Dataset,xr.DataArray)): + hgt = hgt.data + assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) + ny,nx = np.shape(hgt) + + # Get slope and aspect: + hgt_rd = rd.rdarray(hgt, no_data=-9999) + rd.FillDepressions(hgt_rd, in_place=True) + slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun') + aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') + + # Calculate vectors: + vrm = np.zeros((ny,nx)) + rugz = np.cos(slope*np.pi/180.0) + rugdxy = np.sin(slope*np.pi/180.0) + rugx = rugdxy*np.cos(aspect*np.pi/180.0) + rugy = rugdxy*np.sin(aspect*np.pi/180.0) + + # Loop over all cells within bounds of window: + for ii in range(Hwndw+1,nx-Hwndw-1): + for jj in range(Hwndw+1,ny-Hwndw-1): + vrm[jj,ii] = 1.0 - np.sqrt(\ + np.sum(rugx[jj-Hwndw:jj+Hwndw+1,ii-Hwndw:ii+Hwndw+1])**2.0 + \ + np.sum(rugy[jj-Hwndw:jj+Hwndw+1,ii-Hwndw:ii+Hwndw+1])**2.0 + \ + np.sum(rugz[jj-Hwndw:jj+Hwndw+1,ii-Hwndw:ii+Hwndw+1])**2.0)/float(window**2) + return vrm \ No newline at end of file From e4a5fb6f93254091e1f31b0e963004fec5f81052 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 30 Mar 2021 08:33:33 -0600 Subject: [PATCH 057/145] Adding new module to mmctools for VRM Not sure I did this right, but we need richdem v0.3.4 --- environment.yaml | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index 011bb04..95eba4b 100644 --- a/environment.yaml +++ b/environment.yaml @@ -14,3 +14,4 @@ dependencies: - f90nml=1.1.2 - elevation=1.0.6 - rasterio=1.0.25 + - richdem=0.3.4 diff --git a/setup.py b/setup.py index beed2df..efed662 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ 'wrf-python': ['wrf-python>=1.3.2'], # Coupling with terrain (mmctools.coupling.terrain) 'terrain': ['elevation==1.0.6', 'rasterio==1.0.25'], -} + # For calculating vector ruggedness + 'richdem': ['richdem==0.3.4'] # The rest you shouldn't have to touch too much :) # ------------------------------------------------ From 160c5bfa4b3f11d153fd234d3dccb554b468e20f Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 30 Mar 2021 09:42:16 -0600 Subject: [PATCH 058/145] Bug fix - forgot to close dict (}) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index efed662..19d2b28 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ 'terrain': ['elevation==1.0.6', 'rasterio==1.0.25'], # For calculating vector ruggedness 'richdem': ['richdem==0.3.4'] +} # The rest you shouldn't have to touch too much :) # ------------------------------------------------ From 9a96092c09591a1084bef9bf951aa2b433797d16 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 30 Mar 2021 13:20:12 -0600 Subject: [PATCH 059/145] Optimizing TRI calculation Using scipy's generic_filter, this is ~2.5x faster. --- mmctools/helper_functions.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index c44163f..b58e0d8 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1289,7 +1289,8 @@ def calcTRI(hgt,window): window : int Length of window in x and y direction. Must be odd. ''' - + from scipy.ndimage.filters import generic_filter + # Window setup: assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' Hwindow = int(np.floor(window/2)) @@ -1300,12 +1301,21 @@ def calcTRI(hgt,window): assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) ny,nx = np.shape(hgt) - tri = np.zeros((ny,nx)) - # Loop over all cells within bounds of window: - for ii in range(Hwindow+1,nx-Hwindow-1): - for jj in range(Hwindow+1,ny-Hwindow-1): - hgt_window = hgt[jj-Hwindow:jj+Hwindow+1,ii-Hwindow:ii+Hwindow+1] - tri[jj,ii] = (np.sum((hgt_window - hgt[jj,ii])**2.0))**0.5 + + tri_calc_type = 'new' + + if tri_calc_type == 'old': + tri = np.zeros((ny,nx)) + # Loop over all cells within bounds of window: + for ii in range(Hwindow+1,nx-Hwindow-1): + for jj in range(Hwindow+1,ny-Hwindow-1): + hgt_window = hgt[jj-Hwindow:jj+Hwindow+1,ii-Hwindow:ii+Hwindow+1] + tri[jj,ii] = (np.sum((hgt_window - hgt[jj,ii])**2.0))**0.5 + else: + def tri_filt(x): + middle_ind = int(len(x)/2) + return((sum((x - x[middle_ind])**2.0))**0.5) + tri = generic_filter(hgt,tri_filt, size = (3,3)) return tri From a64a16d7bac0cfdd9c29739c2ceedab7da46069a Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 30 Mar 2021 15:57:00 -0600 Subject: [PATCH 060/145] Optimizing TRI and VRM functions --- mmctools/helper_functions.py | 41 ++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index b58e0d8..b8e777d 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1296,26 +1296,18 @@ def calcTRI(hgt,window): Hwindow = int(np.floor(window/2)) # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray)): + if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): hgt = hgt.data assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) ny,nx = np.shape(hgt) - tri_calc_type = 'new' + def tri_filt(x): + middle_ind = int(len(x)/2) + return((sum((x - x[middle_ind])**2.0))**0.5) + + tri = generic_filter(hgt,tri_filt, size = (window,window)) - if tri_calc_type == 'old': - tri = np.zeros((ny,nx)) - # Loop over all cells within bounds of window: - for ii in range(Hwindow+1,nx-Hwindow-1): - for jj in range(Hwindow+1,ny-Hwindow-1): - hgt_window = hgt[jj-Hwindow:jj+Hwindow+1,ii-Hwindow:ii+Hwindow+1] - tri[jj,ii] = (np.sum((hgt_window - hgt[jj,ii])**2.0))**0.5 - else: - def tri_filt(x): - middle_ind = int(len(x)/2) - return((sum((x - x[middle_ind])**2.0))**0.5) - tri = generic_filter(hgt,tri_filt, size = (3,3)) return tri @@ -1333,13 +1325,14 @@ def calcVRM(hgt,window): Length of window in x and y direction. Must be odd. ''' import richdem as rd + from scipy.ndimage.filters import generic_filter # Window setup: assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' Hwndw = int(np.floor(window/2)) # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray)): + if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): hgt = hgt.data assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) ny,nx = np.shape(hgt) @@ -1356,12 +1349,14 @@ def calcVRM(hgt,window): rugdxy = np.sin(slope*np.pi/180.0) rugx = rugdxy*np.cos(aspect*np.pi/180.0) rugy = rugdxy*np.sin(aspect*np.pi/180.0) - - # Loop over all cells within bounds of window: - for ii in range(Hwndw+1,nx-Hwndw-1): - for jj in range(Hwndw+1,ny-Hwndw-1): - vrm[jj,ii] = 1.0 - np.sqrt(\ - np.sum(rugx[jj-Hwndw:jj+Hwndw+1,ii-Hwndw:ii+Hwndw+1])**2.0 + \ - np.sum(rugy[jj-Hwndw:jj+Hwndw+1,ii-Hwndw:ii+Hwndw+1])**2.0 + \ - np.sum(rugz[jj-Hwndw:jj+Hwndw+1,ii-Hwndw:ii+Hwndw+1])**2.0)/float(window**2) + + + def vrm_filt(x): + return(sum(x)**2) + + vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) + vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) + vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) + + vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/float(window**2) return vrm \ No newline at end of file From 826251a643976261b9d66436368460357e154bda Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 31 Mar 2021 14:58:34 -0600 Subject: [PATCH 061/145] Bug fix - dimension / coordinate issue resolved Most of the tests were with xr.Datasets converted to pd.dataframe before being fed into the calc_spectra code. I didn't realize that when you convert xr.Dataset().to_dataframe().to_xarray() that you do not get the original xr.Dataset back... dimension and coordinate changes occur and errors were thrown. The fix makes it so that when a xr.Dataset is read in that each dimension has a coordinate and any coordinates without a dimension become variables --- mmctools/helper_functions.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index b8e777d..0ec2ee1 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1166,7 +1166,14 @@ def calc_spectra(data, data = data.to_dataset() else: raise ValueError('unsupported type: {}'.format(type(data))) - + + for xr_cor in list(data.coords): + if xr_cor not in list(data.dims): + data = data.reset_coords(xr_cor) + for xr_dim in list(data.dims): + if xr_dim not in list(data.coords): + data = data.assign_coords({xr_dim:np.arange(len(data[xr_dim]))}) + # Get index for frequency / wavelength: spec_index = data.coords[spectra_dim] dX = (spec_index.data[1] - spec_index.data[0]) @@ -1226,7 +1233,10 @@ def calc_spectra(data, for ll,lvl in enumerate(level): if lvl is not None: - spec_dat_lvl = data.sel({level_dim:lvl},method='nearest') + if level_dim in list(data.coords.keys()): + spec_dat_lvl = data.sel({level_dim:lvl},method='nearest') + else: + spec_dat_lvl = data.sel({level_dim:lvl}) lvl = spec_dat_lvl[level_dim].data else: spec_dat_lvl = data.copy() @@ -1311,7 +1321,7 @@ def tri_filt(x): return tri -def calcVRM(hgt,window): +def calcVRM(hgt,window,return_slope=False): ''' Vector Ruggedness Measure Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). @@ -1359,4 +1369,7 @@ def vrm_filt(x): vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/float(window**2) - return vrm \ No newline at end of file + if return_slope: + return vrm,slope + else: + return vrm From 568e433fe25be0d6dbf16d05ba95e290895d6522 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 31 Mar 2021 15:02:48 -0600 Subject: [PATCH 062/145] Original workaround no longer needed. --- mmctools/helper_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 0ec2ee1..9ed160c 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1233,10 +1233,7 @@ def calc_spectra(data, for ll,lvl in enumerate(level): if lvl is not None: - if level_dim in list(data.coords.keys()): - spec_dat_lvl = data.sel({level_dim:lvl},method='nearest') - else: - spec_dat_lvl = data.sel({level_dim:lvl}) + spec_dat_lvl = data.sel({level_dim:lvl},method='nearest') lvl = spec_dat_lvl[level_dim].data else: spec_dat_lvl = data.copy() From 47d0b07f6e8f20fd7e1f20cc8362627c83b0a39a Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 1 Apr 2021 12:52:06 -0600 Subject: [PATCH 063/145] Cleaning up fix for dimensions error. --- mmctools/helper_functions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 9ed160c..1085a12 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1167,9 +1167,6 @@ def calc_spectra(data, else: raise ValueError('unsupported type: {}'.format(type(data))) - for xr_cor in list(data.coords): - if xr_cor not in list(data.dims): - data = data.reset_coords(xr_cor) for xr_dim in list(data.dims): if xr_dim not in list(data.coords): data = data.assign_coords({xr_dim:np.arange(len(data[xr_dim]))}) From 682f063724f89d0d3a238125a93d534b04c64d68 Mon Sep 17 00:00:00 2001 From: ewquon Date: Thu, 1 Apr 2021 14:42:21 -0600 Subject: [PATCH 064/145] Update assign_coords calls for backwards compatibility --- mmctools/helper_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 1085a12..55f794c 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1258,7 +1258,7 @@ def calc_spectra(data, tstart=tstart,interval=interval) psd = psd.to_xarray() if avg_dim is not None: - psd = psd.assign_coords({average_dim:1}) + psd = psd.assign_coords(**{average_dim:1}) psd[average_dim] = avg_dim.data psd = psd.expand_dims(average_dim) @@ -1270,7 +1270,7 @@ def calc_spectra(data, psd_level = psd if level_dim is not None: - psd_level = psd_level.assign_coords({level_dim:1}) + psd_level = psd_level.assign_coords(**{level_dim:1}) psd_level[level_dim] = lvl#.data psd_level = psd_level.expand_dims(level_dim) From 5a63735894cb4b21bea454f38eb040063a10fe72 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Thu, 1 Apr 2021 16:24:02 -0600 Subject: [PATCH 065/145] Modifying the geog data path --- mmctools/wrf/preprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 1d3330d..ad7d63f 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -598,7 +598,7 @@ def write_wps_namelist(self): f.write(" truelat1 = {},\n".format(self.namelist_opts['true_lat1'])) f.write(" truelat2 = {},\n".format(self.namelist_opts['true_lat2'])) f.write(" stand_lon = {},\n".format(self.namelist_opts['stand_lon'])) - f.write(" geog_data_path = '/glade/work/hawbecke/geog/',\n") + f.write(" geog_data_path = '{}',\n".format(self.namelist_opts['geog_data_path'])) f.write("/\n") f.write("\n") f.write("&ungrib\n") From 379e1db9dea74a03c6be582c530ecb592a688005 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 6 Apr 2021 12:08:15 -0600 Subject: [PATCH 066/145] Updating TRI and VRM calculations for footprint Using a footprint in the generic filter allows for interesting techniques in calculating TRI / VRM. In this case, we wanted to sub-sample the terrain as it was interpolated (smoothed) and the TRI / VRM values were very low compared to the native terrain data. This footprint allows for sampling a window in only some cells and not others. --- mmctools/helper_functions.py | 40 +++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 1085a12..3d36f6a 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1281,7 +1281,7 @@ def calc_spectra(data, return(psd_f) -def calcTRI(hgt,window): +def calcTRI(hgt,window=None,footprint=None): ''' Terrain Ruggedness Index Riley, S. J., DeGloria, S. D., & Elliot, R. (1999). Index that @@ -1296,6 +1296,10 @@ def calcTRI(hgt,window): from scipy.ndimage.filters import generic_filter # Window setup: + if footprint is not None: + assert window is None, 'Must specify either window or footprint' + window = np.shape(footprint)[0] + assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' Hwindow = int(np.floor(window/2)) @@ -1310,12 +1314,15 @@ def tri_filt(x): middle_ind = int(len(x)/2) return((sum((x - x[middle_ind])**2.0))**0.5) - tri = generic_filter(hgt,tri_filt, size = (window,window)) + if footprint is None: + tri = generic_filter(hgt,tri_filt, size = (window,window)) + else: + tri = generic_filter(hgt,tri_filt, footprint=footprint) return tri -def calcVRM(hgt,window,return_slope=False): +def calcVRM(hgt,window=None,footprint=None,slope_zscale=1.0,return_slope=False): ''' Vector Ruggedness Measure Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). @@ -1330,8 +1337,12 @@ def calcVRM(hgt,window,return_slope=False): ''' import richdem as rd from scipy.ndimage.filters import generic_filter - + # Window setup: + if footprint is not None: + assert window is None, 'Must specify either window or footprint' + window = np.shape(footprint)[0] + assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' Hwndw = int(np.floor(window/2)) @@ -1344,7 +1355,7 @@ def calcVRM(hgt,window,return_slope=False): # Get slope and aspect: hgt_rd = rd.rdarray(hgt, no_data=-9999) rd.FillDepressions(hgt_rd, in_place=True) - slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun') + slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun', zscale=slope_zscale) aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') # Calculate vectors: @@ -1353,16 +1364,25 @@ def calcVRM(hgt,window,return_slope=False): rugdxy = np.sin(slope*np.pi/180.0) rugx = rugdxy*np.cos(aspect*np.pi/180.0) rugy = rugdxy*np.sin(aspect*np.pi/180.0) - def vrm_filt(x): return(sum(x)**2) - vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) - vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) - vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) + if footprint is None: + vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) + vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) + vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) + else: + vrmX = generic_filter(rugx,vrm_filt, footprint=footprint) + vrmY = generic_filter(rugy,vrm_filt, footprint=footprint) + vrmZ = generic_filter(rugz,vrm_filt, footprint=footprint) + - vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/float(window**2) + if footprint is not None: + num_points = len(footprint[footprint != 0.0]) + else: + num_points = float(window**2) + vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/num_points if return_slope: return vrm,slope else: From b5201cec0216ababcf474e4970e216cf3674e280 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Mon, 12 Apr 2021 11:30:45 -0600 Subject: [PATCH 067/145] Allow specptras to be calculated for a list of variables Now a list of variables of interest `var_oi=['u', 'v']` can be given to `calc_spectra` and the spectras will be calculated for all the variables --- mmctools/helper_functions.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 42b73a7..9b036f1 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1102,8 +1102,8 @@ def calc_spectra(data, ========== data : xr.Dataset, xr.DataArray, or pd.dataframe The data that spectra should be calculated over - var_oi : str - Variable of interest - what variable should PSD be computed from. + var_oi : str, or list + Variable(s) of interest - what variable(s) should PSD be computed from. spectra_dim : str Name of the dimension that the variable spans for spectra to be computed. E.g., if you want time spectra, this should be something like @@ -1245,10 +1245,11 @@ def calc_spectra(data, assert len(dim_list) == 1, 'There are too many dimensions... drop one of {}'.format(dim_list) assert len(spec_dat[dim_list[0]].data) == 1, 'Not sure how to parse this dimension, {}, reduce to 1 or remove'.format(dim_list) spec_dat = spec_dat.squeeze() - for varn in list(spec_dat.variables.keys()): - if (varn != var_oi) and (varn != spectra_dim): - spec_dat = spec_dat.drop(varn) - + varsToDrop = set(spec_dat.variables.keys()) \ + - set([spectra_dim] if type(spectra_dim) is str else spectra_dim) \ + - set([var_oi] if type(var_oi) is str else var_oi) + spec_dat = spec_dat.drop(list(varsToDrop)) + spec_dat_df = spec_dat[var_oi].to_dataframe() psd = power_spectral_density(spec_dat_df, From c1832efbf1587c9619eae08f709327d96e822319 Mon Sep 17 00:00:00 2001 From: ewquon Date: Thu, 15 Apr 2021 13:01:40 -0600 Subject: [PATCH 068/145] Recognize 't' as time dimension --- mmctools/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/plotting.py b/mmctools/plotting.py index 7caee31..884c612 100644 --- a/mmctools/plotting.py +++ b/mmctools/plotting.py @@ -75,7 +75,7 @@ # Supported dimensions and associated names dimension_names = { - 'time': ['datetime','time','Time'], + 'time': ['datetime','time','Time','t'], 'height': ['height','heights','z'], 'frequency': ['frequency','f',] } From 2834f3c8cf26c26fff045b3a5078122dd85bf76c Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Mon, 26 Apr 2021 14:17:19 -0600 Subject: [PATCH 069/145] Squashed 'mmctools/windtools/' content from commit c8533ba git-subtree-dir: mmctools/windtools git-subtree-split: c8533ba10e6954e9ec5f66b7d0226fc9db1623f3 --- .gitignore | 2 + LICENSE | 201 ++ README.md | 10 + bin/sowfa_convert_source_history.py | 79 + bin/sowfa_extract_elevation_from_stl.py | 49 + bin/sowfa_log_to_df.py | 33 + setup.py | 108 + windtools/SOWFA6/constant/boundaryData.py | 623 +++++ windtools/SOWFA6/log.py | 115 + windtools/SOWFA6/postProcessing/averaging.py | 117 + windtools/SOWFA6/postProcessing/probeSets.py | 308 +++ windtools/SOWFA6/postProcessing/probes.py | 138 ++ windtools/SOWFA6/postProcessing/reader.py | 351 +++ .../SOWFA6/postProcessing/sourceHistory.py | 124 + windtools/common.py | 190 ++ windtools/inflow/general.py | 517 ++++ windtools/inflow/synthetic.py | 271 +++ windtools/io/binary.py | 122 + windtools/io/ensight.py | 47 + windtools/io/series.py | 232 ++ windtools/io/vtk.py | 171 ++ windtools/openfast.py | 130 + windtools/openfoam.py | 294 +++ windtools/plotting.py | 2154 +++++++++++++++++ 24 files changed, 6386 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 bin/sowfa_convert_source_history.py create mode 100755 bin/sowfa_extract_elevation_from_stl.py create mode 100755 bin/sowfa_log_to_df.py create mode 100644 setup.py create mode 100644 windtools/SOWFA6/constant/boundaryData.py create mode 100644 windtools/SOWFA6/log.py create mode 100644 windtools/SOWFA6/postProcessing/averaging.py create mode 100644 windtools/SOWFA6/postProcessing/probeSets.py create mode 100644 windtools/SOWFA6/postProcessing/probes.py create mode 100644 windtools/SOWFA6/postProcessing/reader.py create mode 100644 windtools/SOWFA6/postProcessing/sourceHistory.py create mode 100644 windtools/common.py create mode 100644 windtools/inflow/general.py create mode 100644 windtools/inflow/synthetic.py create mode 100644 windtools/io/binary.py create mode 100644 windtools/io/ensight.py create mode 100644 windtools/io/series.py create mode 100644 windtools/io/vtk.py create mode 100644 windtools/openfast.py create mode 100644 windtools/openfoam.py create mode 100644 windtools/plotting.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8d35cb3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +*.pyc diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6f75635 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5eb5d9c --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# windtools + +Python tools for wind simulation setup, data processing, and analysis. + +Maintained by [Eliot Quon](https://github.com/ewquon) () + +These tools are intended to support research with the following codes: +- Simulator fOr Wind Farm Applications (https://github.com/NREL/SOWFA-6/tree/dev) +- OpenFAST (https://openfast.readthedocs.io) +- More to come! diff --git a/bin/sowfa_convert_source_history.py b/bin/sowfa_convert_source_history.py new file mode 100755 index 0000000..6c48bce --- /dev/null +++ b/bin/sowfa_convert_source_history.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# Copyright 2020 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +""" +Convert precursor sourceHistory into OpenFOAM dictionaries to include in +constant/ABLProperties as _given_ momentum and/or temperature source +terms. + +USAGE: sowfa_convert_source_history.py [/path/to/postProcessing/sourceHistory] + +where the default path is ./postProcessing/sourceHistory. + +Upon completion, the script will write out: +- constant/givenSourceU +- constant/givenSourceT + +""" +import sys, os +import numpy as np +from windtools.SOWFA6.postProcessing.sourceHistory import SourceHistory + +try: + srchistpath = sys.argv[1] +except IndexError: + srchistpath = 'postProcessing/sourceHistory' +if not os.path.isdir(srchistpath): + print(srchistpath,'not found; please specify a valid path') + +sourceU = SourceHistory(srchistpath,varList='Momentum') +sourceT = SourceHistory(srchistpath,varList='Temperature') + +outpath = 'constant' +try: + open(os.path.join(outpath,'givenSourceU'),'w') +except IOError: + outpath = '.' + +sourceUpath = os.path.join(outpath,'givenSourceU') +with open(sourceUpath,'w') as f: + fmt = ' %g' + f.write('sourceHeightsMomentum\n') + f.write('(\n') + np.savetxt(f, sourceU.hLevelsCell, fmt=fmt) + f.write(');\n\n') + + fmt = ' (' + ' '.join((sourceU.N+1)*['%g']) + ')' + for i,comp in enumerate(['X','Y','Z']): + srcdata = np.hstack((sourceU.t[:,np.newaxis], sourceU.Momentum[:,:,i])) + f.write(f'sourceTableMomentum{comp}\n') + f.write('(\n') + np.savetxt(f, srcdata, fmt=fmt) + f.write(');\n\n') +print('Wrote',sourceUpath) + +sourceTpath = os.path.join(outpath,'givenSourceT') +with open(sourceTpath,'w') as f: + fmt = ' %g' + f.write('sourceHeightsTemperature\n') + f.write('(\n') + np.savetxt(f, sourceT.hLevelsCell, fmt=fmt) + f.write(');\n\n') + + fmt = ' (' + ' '.join((sourceT.N+1)*['%g']) + ')' + srcdata = np.hstack((sourceT.t[:,np.newaxis], sourceT.Temperature)) + f.write(f'sourceTableTemperature\n') + f.write('(\n') + np.savetxt(f, srcdata, fmt=fmt) + f.write(');\n\n') +print('Wrote',sourceTpath) + diff --git a/bin/sowfa_extract_elevation_from_stl.py b/bin/sowfa_extract_elevation_from_stl.py new file mode 100755 index 0000000..af07b8d --- /dev/null +++ b/bin/sowfa_extract_elevation_from_stl.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2020 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import sys, glob +import numpy as np +from scipy.interpolate import griddata +# pip install numpy-stl +from stl import mesh + +stlpath = glob.glob('constant/triSurface/*.stl') +interp_method = 'cubic' # linear, nearest, cubic + +if (len(sys.argv) < 2): + sys.exit('USAGE: '+sys.argv[0]+' x,y [x,y] ...') +assert (len(stlpath) == 1), 'Did not find single stl file in constant/triSurface' + +print('Reading stl from',stlpath[0]) +msh = mesh.Mesh.from_file(stlpath[0]) +x = msh.vectors[:,:,0].ravel() +y = msh.vectors[:,:,1].ravel() +z = msh.vectors[:,:,2].ravel() + +# construct output points +xout = [] +yout = [] +for xy in sys.argv[1:]: + assert (',' in xy) + xyvals = [ float(val) for val in xy.split(',') ] + assert (len(xyvals) == 2) + xout.append(xyvals[0]) + yout.append(xyvals[1]) + +points = np.stack((x,y), axis=-1) +xi = np.stack((xout,yout), axis=-1) +elev = griddata(points, z, xi, method=interp_method) + +for xo,yo,zo in zip(xout,yout,elev): + print(xo,yo,zo) + diff --git a/bin/sowfa_log_to_df.py b/bin/sowfa_log_to_df.py new file mode 100755 index 0000000..83c66f1 --- /dev/null +++ b/bin/sowfa_log_to_df.py @@ -0,0 +1,33 @@ +# Copyright 2020 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import sys +import pandas as pd +from windtools.SOWFA6.log import LogFile + +if len(sys.argv) <= 1: + sys.exit('Specify log file(s)') + +logfiles = sys.argv[1:] +if len(logfiles) > 1: + outfile = 'combined_log.csv' +else: + outfile = logfiles[0] + '.csv' + +print('Scraping log files:',logfiles) +df = pd.concat([LogFile(fpath).df for fpath in logfiles]) + +# drop duplicate rows (for restarts) +df.drop_duplicates(keep='last',inplace=True) + +print('Writing',outfile) +df.to_csv(outfile) + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..afa9c41 --- /dev/null +++ b/setup.py @@ -0,0 +1,108 @@ +""" +Copyright 2017 NREL + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +# This setup file is based on https://github.com/NREL/floris/blob/master/setup.py +# accessed on April 3, 2020. + +# Note: To use the 'upload' functionality of this file, you must: +# $ pip install twine + +import io +import os +import glob + +from setuptools import find_packages, setup + +# Package meta-data. +NAME = 'windtools' +DESCRIPTION = 'Python tools for wind simulation setup, data processing, and analysis' +URL = 'https://github.com/NREL/windtools' +EMAIL = 'eliot.quon@nrel.gov' +AUTHOR = 'U.S. Department of Energy' +REQUIRES_PYTHON = '>=3.6.0' +VERSION = '0.1.0' + +# What packages are required for this module to be executed? +REQUIRED = [ + 'matplotlib>=3', + 'numpy>=1.18.1', + 'scipy>=1.4.1', + 'pandas>=1.0.1', + 'xarray>=0.15.0', +] + +EXTRAS = {} + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +try: + with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() +except FileNotFoundError: + long_description = DESCRIPTION + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + project_slug = NAME.lower().replace("-", "_").replace(" ", "_") + with open(os.path.join(here, project_slug, '__version__.py')) as f: + exec(f.read(), about) +else: + about['__version__'] = VERSION + +# Get executable scripts +scripts = glob.glob(os.path.join(here, 'bin', '*')) + +# Where the magic happens: +setup( + name=NAME, + version=about['__version__'], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type='text/markdown', + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + #packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]), + # If your package is a single module, use this instead of 'packages': + py_modules=[NAME], + # entry_points={ + # 'console_scripts': ['mycli=mymodule:cli'], + # }, + scripts=scripts, + install_requires=REQUIRED, + extras_require=EXTRAS, + include_package_data=True, + license='Apache-2.0', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: Apache Software License', + 'Topic :: Scientific/Engineering', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Development Status :: 4 - Beta', + ], + # $ setup.py publish support. + #cmdclass={ + # 'upload': UploadCommand, + #}, +) diff --git a/windtools/SOWFA6/constant/boundaryData.py b/windtools/SOWFA6/constant/boundaryData.py new file mode 100644 index 0000000..6284454 --- /dev/null +++ b/windtools/SOWFA6/constant/boundaryData.py @@ -0,0 +1,623 @@ +# +# Module to handle SOWFA boundary data that belong in +# casedir/constant/boundaryData +# +# Written by Eliot Quon (eliot.quon@nrel.gov) +# +import os +import numpy as np +import matplotlib.pyplot as plt + +from windtools.io.series import TimeSeries + +contour_colormap = 'RdBu_r' # more soothing blues and reds + +pointsheader = """/*--------------------------------*- C++ -*----------------------------------*\\ +| ========= | | +| \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\\\ / O peration | Version: 2.4.x | +| \\\\ / A nd | Web: www.OpenFOAM.org | +| \\\\/ M anipulation | | +\\*---------------------------------------------------------------------------*/ +FoamFile +{{ + version 2.0; + format {fmt:s}; + class vectorField; + location "constant/boundaryData/{patchName:s}"; + object points; +}} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +{N:d} +(""" + +dataheader = """/*--------------------------------*- C++ -*----------------------------------*\\ +| ========= | | +| \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\\\ / O peration | Version: 2.4.x | +| \\\\ / A nd | Web: www.OpenFOAM.org | +| \\\\/ M anipulation | | +\\*---------------------------------------------------------------------------*/ +FoamFile +{{ + version 2.0; + format {fmt:s}; + class {patchType:s}AverageField; + location "constant/boundaryData/{patchName:s}/{timeName:s}"; + object values; +}} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Average +{avgValue:s} + +{N:d} +(""" + + +def write_points(fname,x,y,z,patchName='patch', + fmt='%f',order='C'): + """Write out a points file which should be stored in + constant/boundaryData/patchName/points + """ + N = len(x) + assert(N == len(y) == len(z)) + if len(x.shape) > 1: + x = x.ravel(order=order) + y = y.ravel(order=order) + z = z.ravel(order=order) + N = len(x) + dpath = os.path.split(fname)[0] + if not os.path.isdir(dpath): + os.makedirs(dpath) + fmtstr = '({:s} {:s} {:s})'.format(fmt,fmt,fmt) + np.savetxt(fname, + np.stack((x,y,z)).T, fmt=fmtstr, + header=pointsheader.format(patchName=patchName,N=N,fmt='ascii'), + footer=')', + comments='') + +def write_data(fname, + data, + patchName='patch', + timeName=0, + avgValue=None): + """Write out a boundaryData file which should be stored in + constant/boundarydata/patchName/timeName/fieldName + + Parameters + ---------- + fname : str + Output data file name + data : numpy.ndarray + Field data to be written out, with shape (3,N) for vectors + and shape (N) for scalars; 2-D or 3-D data should be flattened + beforehand. + patchName : str, optional + Name of the boundary patch + timeName : scalar or str, optional + Name of corresponding time directory + avgValue : scalar or list-like, optional + To set avgValue in the data file; probably not used. + + @author: ewquon + """ + dims = data.shape + N = dims[-1] + if len(dims) == 1: + patchType = 'scalar' + if avgValue is None: + avgValueStr = '0' + else: + avgValueStr = str(avgValue) + elif len(dims) == 2: + patchType = 'vector' + assert(dims[0] == 3) + if avgValue is None: + avgValueStr = '(0 0 0)' + else: + avgValueStr = '(' + ' '.join([str(val) for val in list(avgValue)]) + ')' + else: + print('ERROR: Unexpected number of dimensions! No data written.') + return + + dpath = os.path.split(fname)[0] + if not os.path.isdir(dpath): + os.makedirs(dpath) + + headerstr = dataheader.format(patchType=patchType, + patchName=patchName, + timeName=str(timeName), + avgValue=avgValueStr, + N=N, + fmt='ascii') + if patchType == 'vector': + np.savetxt(fname, + data.T, fmt='(%g %g %g)', + header=headerstr, footer=')', + comments='') + elif patchType == 'scalar': + np.savetxt(fname, + data.reshape((N,1)), fmt='%g', + header=headerstr, footer=')', + comments='') + + +def get_unique_points_from_list(ylist,zlist,NY=None,NZ=None,order='F'): + """Detects y and z (1-D arrays) from a list of points on a + structured grid. Makes no assumptions about the point + ordering + """ + ylist = np.array(ylist) + zlist = np.array(zlist) + N = len(zlist) + assert(N == len(ylist)) + if (NY is not None) and (NZ is not None): + # use specified plane dimensions + assert(NY*NZ == N) + y = ylist.reshape((NY,NZ))[:,0] + elif zlist[1]==zlist[0]: + # y changes faster, F-ordering + NY = np.nonzero(zlist > zlist[0])[0][0] + NZ = int(N / NY) + assert(NY*NZ == N) + y = ylist[:NY] + z = zlist.reshape((NY,NZ),order='F')[0,:] + elif ylist[1]==ylist[0]: + # z changes faster, C-ordering + NZ = np.nonzero(ylist > ylist[0])[0][0] + NY = int(N / NZ) + assert(NY*NZ == N) + z = zlist[:NZ] + y = ylist.reshape((NY,NZ),order='C')[:,0] + else: + print('Unrecognized point distribution') + print('"y" :',len(ylist),ylist) + print('"z" :',len(zlist),zlist) + return ylist,zlist,False + return y,z,True + + +def read_points(fname,tol=1e-6,return_const=False,**kwargs): + """Returns a 2D set of points if one of the coordinates is constant + otherwise returns a 3D set of points. Assumes that the points are on a + structured grid. + """ + N = None + points = None + iread = 0 + with open(fname,'r') as f: + while N is None: + try: + N = int(f.readline()) + except ValueError: pass + else: + points = np.zeros((N,3)) + print('Reading',N,'points from',fname) + for line in f: + line = line[:line.find('\\')].strip() + try: + points[iread,:] = [ float(val) for val in line[1:-1].split() ] + except (ValueError, IndexError): pass + else: + iread += 1 + assert(iread == N) + + #constX = np.all(points[:,0] == points[0,0]) + #constY = np.all(points[:,1] == points[0,1]) + #constZ = np.all(points[:,2] == points[0,2]) + constX = np.max(points[:,0]) - np.min(points[0,0]) < tol + constY = np.max(points[:,1]) - np.min(points[0,1]) < tol + constZ = np.max(points[:,2]) - np.min(points[0,2]) < tol + print('Constant in x/y/z :',constX,constY,constZ) + if not (constX or constY): + print('Warning: boundary is not constant in X or Y?') + + if constX: + x0 = np.mean(points[:,0]) + ylist = points[:,1] + zlist = points[:,2] + elif constY: + x0 = np.mean(points[:,1]) + ylist = points[:,0] + zlist = points[:,2] + elif constZ: + x0 = np.mean(points[:,2]) + ylist = points[:,0] + zlist = points[:,1] + else: + print('Unexpected boundary orientation, returning full list of points') + return points + + y,z,is_structured = get_unique_points_from_list(ylist,zlist,**kwargs) + assert(is_structured) + if return_const: + return x0,y,z + else: + return y,z + + +def read_vector_data(fname,Ny=None,Nz=None,order='C',verbose=False): + """Read vector field data from a structured boundary data patch""" + N = None + data = None + iread = 0 + with open(fname,'r') as f: + for line in f: + if N is None: + try: + N = int(line) + if (Ny is not None) and (Nz is not None): + if not N == Ny*Nz: + Ny = None + Nz = None + data = np.zeros((N,3)) + if verbose: print('Reading',N,'vectors from',fname) + except ValueError: pass + elif not line.strip() in ['','(',')',';'] \ + and not line.strip().startswith('//'): + data[iread,:] = [ float(val) for val in line.strip().strip('()').split() ] + iread += 1 + assert(iread == N) + + if (Ny is not None) and (Nz is not None): + vectorField = np.zeros((3,Ny,Nz)) + for i in range(3): + vectorField[i,:,:] = data[:,i].reshape((Ny,Nz),order=order) + else: + vectorField = data.T + + return vectorField + + +def read_scalar_data(fname,Ny=None,Nz=None,order='C',verbose=False): + """Read scalar field data from a structured boundary data patch""" + N = None + data = None + iread = 0 + with open(fname,'r') as f: + for line in f: + if (N is None) or N < 0: + try: + if N is None: + avgval = float(line) + N = -1 # skip first scalar, which is the average field value (not used) + else: + assert(N < 0) + N = int(line) # now read the number of points + if (Ny is not None) and (Nz is not None): + if not N == Ny*Nz: + Ny = None + Nz = None + data = np.zeros(N) + if verbose: print('Reading',N,'scalars from',fname) + except ValueError: pass + elif not line.strip() in ['','(',')',';'] \ + and not line.strip().startswith('//'): + data[iread] = float(line) + iread += 1 + assert(iread == N) + + if (Ny is not None) and (Nz is not None): + scalarField = data.reshape((Ny,Nz),order=order) + else: + scalarField = data + + return scalarField + + +class BoundaryData(object): + """Object to handle boundary data""" + + def __init__(self, + bdpath,Ny=None,Nz=None,order='F', + fields=['U','T','k'], + verbose=True): + """Process timeVaryingMapped* boundary data located in in + constant/boundaryData/ + """ + self.dpath = bdpath + assert(os.path.isdir(bdpath)) + self.name = os.path.split(bdpath)[-1] + + self.ts = TimeSeries(bdpath,dirs=True,verbose=verbose) + self.t = np.array(self.ts.times) + self.Ntimes = self.ts.Ntimes + + kwargs = {} + if (Ny is not None) and (Nz is not None): + kwargs = dict(Ny=Ny, Nz=Nz) + self.y, self.z = read_points(os.path.join(bdpath,'points'), **kwargs) + self.Ny = len(self.y) + self.Nz = len(self.z) + + #self.yy, self.zz = np.meshgrid(self.y, self.z, indexing='ij') + dy = np.diff(self.y) + dz = np.diff(self.z) + assert(np.all(dy==dy[0])) # uniform spacing assumed + assert(np.all(dz==dz[0])) + dy = dy[0] + dz = dz[0] + self.yy, self.zz = np.meshgrid(np.arange(self.Ny+1)*dy + self.y[0]-dy/2, + np.arange(self.Nz+1)*dz + self.z[0]-dz/2, + indexing='ij') + + # image left, right, bottom, top (for imshow) + self.extent = (self.y[0], self.y[-1], + self.z[-1], self.z[0]) # note top/bottom flipped + + self.field = {} + haveU, haveT, havek = False, False, False + if 'U' in fields: + haveU = True + self.field['U'] = np.zeros((self.Ntimes,self.Ny,self.Nz)) + self.field['V'] = np.zeros((self.Ntimes,self.Ny,self.Nz)) + self.field['W'] = np.zeros((self.Ntimes,self.Ny,self.Nz)) + if 'T' in fields: + haveT = True + self.field['T'] = np.zeros((self.Ntimes,self.Ny,self.Nz)) + if 'k' in fields: + havek = True + self.field['k'] = np.zeros((self.Ntimes,self.Ny,self.Nz)) + + for itime, dpath in enumerate(self.ts): + if verbose: + print('t={:f} {:s}'.format(self.ts.times[itime],dpath)) + Ufield = read_vector_data(os.path.join(dpath, 'U'), + Ny=self.Ny, Nz=self.Nz) + self.field['U'][itime,:,:] = Ufield[0,:,:] + self.field['V'][itime,:,:] = Ufield[1,:,:] + self.field['W'][itime,:,:] = Ufield[2,:,:] + Tfield = read_scalar_data(os.path.join(dpath, 'T'), + Ny=self.Ny, Nz=self.Nz) + self.field['T'][itime,:,:] = Tfield + kfield = read_scalar_data(os.path.join(dpath, 'k'), + Ny=self.Ny, Nz=self.Nz) + self.field['k'][itime,:,:] = kfield + + def __getattr__(self,name): + if name in self.field.keys(): + return self.field[name] + else: + raise AttributeError("Boundary data do not include field '{:s}'".format(name)) + + + def to_npz(self,fpath='boundaryData.npz'): + np.savez_compressed(fpath,**self.field) + + + def create(self,name,field): + assert(np.all(field.shape == self.field['U'].shape)) + self.field[name] = field + + + def _plot_patch(self,F,name,time,value_range): + fig = plt.figure(1,figsize=(8,6)) + itime = np.argmin(np.abs(time - self.t)) + #print(itime,time) +# F = self.field[field] #[itime,:,:] + if self._init_patch_plot: + # reset range controls + fieldmin = np.min(F) + fieldmax = np.max(F) + minval = fieldmin + maxval = fieldmax + self._update_vrange_minmax(fieldmin,fieldmax) + self.vrange.value = (fieldmin,fieldmax) + self._init_patch_plot = False + else: + minval, maxval = value_range + ax = plt.gca() + cmesh = ax.pcolormesh(self.yy, self.zz, F[itime,:,:], + cmap='RdBu_r', + vmin=minval, vmax=maxval) + + ax.set_aspect('equal','box') + ax.set_title('{:s} (i={:d}: t={:g} s)'.format(name,itime,self.t[itime])) + cbar = plt.colorbar(cmesh,ax=ax,orientation='horizontal',fraction=0.04) + cbar.set_label(name) + + def _update_vrange_minmax(self,fieldmin,fieldmax): + if fieldmin < self.vrange.max: + self.vrange.min = fieldmin + self.vrange.max = fieldmax + else: + self.vrange.max = fieldmax + self.vrange.min = fieldmin + + def iplot(self,name): + from ipywidgets import interactive, fixed #interact, interactive, fixed, interact_manual + import ipywidgets as widgets + from IPython.display import display +# allfields = list(self.field.keys()) + F = self.field[name] + fieldmin = np.min(F) + fieldmax = np.max(F) + timeselector = widgets.FloatSlider( + min=self.ts.times[0], + max=self.ts.times[-1], + step=(self.ts.times[1]-self.ts.times[0]), + value=self.ts.times[0] + ) + self.vrange = widgets.FloatRangeSlider( + min=fieldmin, + max=fieldmax, + step=(fieldmax-fieldmin)/10, + value=(fieldmin,fieldmax) + ) + + self._init_patch_plot = True + self.plotwidget = interactive( + self._plot_patch, + F=fixed(F), + name=fixed(name), + time=timeselector, + value_range=self.vrange, + ) + display(self.plotwidget) + + +class CartesianPatch(object): + """Object to facilitate outputing boundary patches on a Cartesian + grid, with boundaries at x=const or y=const. + """ + + def __init__(self,x,y,z,dpath='.',name='patch'): + """For a Cartesian mesh, the grid is defined by 1-D coordinate + vectors x,y,z + """ + self.dpath = dpath + self.name = name + # check for constant x/y + if np.all(x==x[0]): + self.desc = 'const x={:.1f}'.format(x[0]) + x = [x[0]] + elif np.all(y==y[0]): + self.desc = 'const y={:.1f}'.format(y[0]) + y = [y[0]] + else: + raise ValueError('x and y not constant, domain is not Cartesian') + # set up points + self.x = x + self.y = y + self.z = z + self.Nx = len(x) + self.Ny = len(y) + self.Nz = len(z) + # set up mesh + self.X, self.Y, self.Z = np.meshgrid(x,y,z,indexing='ij') + + def __repr__(self): + s = 'Cartesian patch "{:s}" : '.format(self.name) + s += self.desc + s +=', size=({:d},{:d},{:d})'.format(self.Nx,self.Ny,self.Nz) + return s + + def write_points(self,fpath=None): + """Write out constant/boundaryData/patchName/points file""" + if fpath is None: + fpath = os.path.join(self.dpath,self.name,'points') + write_points(fpath,self.X,self.Y,self.Z,patchName=self.name) + print('Wrote points to '+fpath) + + def write_profiles(self, t, z, + U=None, V=None, W=None, T=None, k=None, + time_range=[None,None], + verbose=True): + """Write out constant/boundaryData/patchName/*/{U,T} given a + set of time-height profiles. Outputs will be interpolated to + the patch heights. + + Inputs + ------ + t : np.ndarray + Time vector with length Nt + z : np.ndarray + Height vector with length Nz + U : np.ndarray + Velocity vectors (Nt,Nz,3) or x-velocity component (Nt,Nz) + V : np.ndarray, optional + y-velocity component (Nt,Nz) + W : np.ndarray, optional + z-velocity component (Nt,Nz) + T : np.ndarray + potential temperature (Nt,Nz) + time_range : tuple, optional + range of times to write out boundary data + """ + assert(U is not None) + assert(T is not None) + Nt, Nz = U.shape[:2] + assert(Nt == len(t)) + assert(Nz == len(z)) + if len(U.shape) == 2: + have_components = True + assert(np.all(U.shape == (Nt,Nz)) and \ + np.all(V.shape == (Nt,Nz)) and \ + np.all(W.shape == (Nt,Nz)) and \ + np.all(T.shape == (Nt,Nz))) + elif len(U.shape) == 3: + have_components = False + assert(U.shape[2] == 3) + assert(np.all(T.shape == (Nt,Nz))) + else: + raise InputError + + if not have_components: + V = U[:,:,1] + W = U[:,:,2] + U = U[:,:,0] + + if k is not None: + assert(np.all(k.shape == (Nt,Nz))) + + if time_range[0] is None: + time_range[0] = 0.0 + if time_range[1] is None: + time_range[1] = 9e9 + + if (not len(self.z) == Nz) or (not np.all(self.z == z)): + interpolate = True + else: + # input z are equal to patch z + interpolate = False + + Upatch = np.zeros((self.Nx,self.Ny,self.Nz)) + Vpatch = np.zeros((self.Nx,self.Ny,self.Nz)) + Wpatch = np.zeros((self.Nx,self.Ny,self.Nz)) + Tpatch = np.zeros((self.Nx,self.Ny,self.Nz)) + kpatch = np.zeros((self.Nx,self.Ny,self.Nz)) + for it,ti in enumerate(t): + if (ti < time_range[0]) or (ti > time_range[1]): + if verbose: + print('skipping time '+str(ti)) + continue + tname = '{:f}'.format(ti).rstrip('0').rstrip('.') + + timepath = os.path.join(self.dpath, self.name, tname) + if not os.path.isdir(timepath): + os.makedirs(timepath) + + Upatch[:,:,:] = 0.0 + Vpatch[:,:,:] = 0.0 + Wpatch[:,:,:] = 0.0 + Tpatch[:,:,:] = 0.0 + kpatch[:,:,:] = 0.0 + if interpolate: + if verbose: + print('interpolating data at t = {:s} s'.format(tname)) + for iz,zp in enumerate(self.z): + i = np.nonzero(z > zp)[0][0] + f = (zp - z[i-1]) / (z[i] - z[i-1]) + Upatch[:,:,iz] = U[it,i-1] + f*(U[it,i]-U[it,i-1]) + Vpatch[:,:,iz] = V[it,i-1] + f*(V[it,i]-V[it,i-1]) + Wpatch[:,:,iz] = W[it,i-1] + f*(W[it,i]-W[it,i-1]) + Tpatch[:,:,iz] = T[it,i-1] + f*(T[it,i]-T[it,i-1]) + if k is not None: + kpatch[:,:,iz] = k[it,i-1] + f*(k[it,i]-k[it,i-1]) + else: + if verbose: + print('mapping data at t = {:s} s'.format(tname)) + for iz in range(Nz): + Upatch[:,:,iz] = U[it,iz] + Vpatch[:,:,iz] = V[it,iz] + Wpatch[:,:,iz] = W[it,iz] + Tpatch[:,:,iz] = T[it,iz] + if k is not None: + kpatch[:,:,iz] = k[it,iz] + + Upath = os.path.join(timepath,'U') + Udata = np.stack((Upatch.ravel(), Vpatch.ravel(), Wpatch.ravel())) + if verbose: print('writing data to {:s}'.format(Upath)) + write_data(Upath, Udata, patchName=self.name, timeName=ti) + + Tpath = os.path.join(timepath,'T') + if verbose: print('writing data to {:s}'.format(Tpath)) + write_data(Tpath, Tpatch.ravel(), patchName=self.name, timeName=ti) + + if k is not None: + kpath = os.path.join(timepath,'k') + if verbose: print('writing data to {:s}'.format(kpath)) + write_data(kpath, kpatch.ravel(), patchName=self.name, timeName=ti) + + diff --git a/windtools/SOWFA6/log.py b/windtools/SOWFA6/log.py new file mode 100644 index 0000000..a3541a1 --- /dev/null +++ b/windtools/SOWFA6/log.py @@ -0,0 +1,115 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import os,sys +import pandas as pd + +class LogFile(object): + """Scrape SOWFA log file for simulation information and store in a + pandas dataframe. + """ + def __init__(self, fpath): + self._read(fpath) + + def _read(self, fpath): + if not os.path.isfile(fpath): + sys.exit(fpath,'not found') + startedTimeLoop = False + times = [] + dt = [] + CoMean = [] + CoMax = [] + contErrMin = [] + contErrMax = [] + contErrMean = [] + bndryFluxTot = [] + turbine_thrust = [] + with open(fpath,'r') as f: + for line in f: + line = line.strip() + if line.startswith('Create mesh'): + startTime = float(line.split()[-1]) + print('Simulation start from t=',startTime) + elif line.startswith('Starting time loop'): + startedTimeLoop = True + if not startedTimeLoop: + continue + if line.startswith('Time ='): + # Time = 0.5 Time Step = 1 + curTime = float(line.split()[2]) + times.append(curTime) + elif line.startswith('deltaT'): + # deltaT = 0.5 + dt.append(float(line.split()[2])) + elif line.startswith('Courant Number'): + # Courant Number mean: 0.25 max: 0.25 + line = line.split() + CoMean.append(float(line[3])) + CoMax.append(float(line[5])) + elif line.startswith('minimum:'): + # -Local Cell Continuity Error: + # minimum: 4.78677122625e-18 + # maximum: 4.1249975169e-10 + # weighted mean: 1.55849978382e-12 + contErrMin.append(float(line.split()[1])) + elif line.startswith('maximum:'): + contErrMax.append(float(line.split()[1])) + elif line.startswith('weighted mean:'): + contErrMean.append(float(line.split()[2])) + elif line.startswith('total - flux:'): + # -Boundary Flux: + # lower - flux: 0 / area: 1000000 + # upper - flux: -3.40510497274e-19 / area: 1000000 + # west - flux: -4972611.5 / area: 500000 + # east - flux: 4971897.53527 / area: 500000 + # north - flux: 49044.2606921 / area: 500000 + # south - flux: -48330.2959591 / area: 500000 + # total - flux: 3.52156348526e-09 / area: 4000000 + bndryFluxTot.append(float(line.split()[3])) + elif line.startswith('Turbine'): + # Turbine 0 Rotor Torque from Body Force = 80826.6608129 Rotor Torque from Actuator = 80826.6608129 Ratio = 1 + # Turbine 0 Rotor Axial Force from Body Force = 31188.1287252 Rotor Axial Force from Actuator = 31188.1287237 Ratio = 1.00000000005 + # Turbine 0 Nacelle Axial Force from BodyForce = 154.910049102 Nacelle Axial Force from Actuator = 153.712783319 Ratio = 1.00778897993 + line = line.split() + iturb = int(line[1]) + if ' '.join(line[2:5]) == 'Rotor Axial Force': + thrust = float(line[9]) + try: + turbine_thrust[iturb].append(thrust) + except IndexError: + turbine_thrust.append([thrust]) + assert (len(turbine_thrust) == iturb+1) + + # create data dict + data = {} + if len(dt) > 0: + # adjustTimeStep is on + data['deltaT'] = dt + data['CoMean'] = CoMean + data['CoMax'] = CoMax + data['continuityErrorMin'] = contErrMin + data['continuityErrorMax'] = contErrMax + data['continuityErrorWeightedMean'] = contErrMean + data['boundaryFluxTotal'] = bndryFluxTot + for iturb,thrust in enumerate(turbine_thrust): + data[f'thrust{iturb:d}'] = thrust + + # trim data if needed + datalengths = [len(arr) for _,arr in data.items()] + if not all([N == datalengths[0] for N in datalengths]): + Nsteps = min(datalengths) + times = times[:Nsteps] + for name,arr in data.items(): + data[name] = arr[:Nsteps] + print('Note: Truncated number of steps to',Nsteps) + + self.df = pd.DataFrame(data, index=pd.Index(times, name='time')) + diff --git a/windtools/SOWFA6/postProcessing/averaging.py b/windtools/SOWFA6/postProcessing/averaging.py new file mode 100644 index 0000000..ef6aa07 --- /dev/null +++ b/windtools/SOWFA6/postProcessing/averaging.py @@ -0,0 +1,117 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +For processing SOWFA-6 planar averages + +Based on original SOWFA/postProcessing/averaging.py from github.com/NWTC/datatools + +written by +- Dries Allaerts (dries.allaerts@nrel.gov) +- Eliot Quon (eliot.quon@nrel.gov) + +Sample usage: + + from windtools.SOWFA6.postProcessing.averaging import PlanarAverages + + # read all time directories in current working directory + averagingData = PlanarAverages() + + # read '0' and '1000' in current working directory + averagingData = planarAverages( 0, 1000 ) + + # read all time directories in specified directory + averagingData = PlanarAverages('caseX/postProcessing/averaging') + + # read specified time directories + averagingData = PlanarAverages('caseX/postProcessing/averaging/0', + 'caseX/postProcessing/averaging/1000') + +""" +from __future__ import print_function +import os +import numpy as np +import matplotlib.pyplot as plt +from matplotlib import cm +from .reader import Reader + +class PlanarAverages(Reader): + + def __init__(self,dpath=None,**kwargs): + super().__init__(dpath,includeDt=True,**kwargs) + + + def _processdirs(self, + tdirList, + varList=['U','T'], + trimOverlap=True + ): + #Redefine _processdirs so that default + #argument for varList can be specified + super()._processdirs(tdirList,varList,trimOverlap) + + + def _read_data(self,dpath,fname): + fpath = dpath + os.sep + fname + with open(fpath) as f: + try: + self._read_heights(f) + except IOError: + print('unable to read '+fpath) + else: + array = self._read_field_data(f) + return array + + def _read_heights(self,f): + line = f.readline().split() + assert (line[0] == 'Heights'), \ + 'Error: Expected first line to start with "Heights", but instead read'+line[0] + + self.hLevelsCell = [ float(val) for val in line[2:] ] + f.readline() + + if (len(self._processed) > 0): # assert that all fields have same number of heights + assert (self.N == len(self.hLevelsCell)), \ + 'Error: Various fields do not have the same number of heights' + else: # first field: set number of heights in self.N + self.N = len(self.hLevelsCell) + self.hLevelsCell = np.array(self.hLevelsCell) + + + def _read_field_data(self,f): + out = [] + for line in f: + line = [ float(val) for val in + line.replace('(','').replace(')','').split() ] + out.append(line) + return np.array(out) + + + #============================================================================ + # + # DATA I/O + # + #============================================================================ + + def to_netcdf(self,fname): + fieldDescriptions = {'T': 'Potential temperature', + 'Ux': 'U velocity component', + 'Uy': 'V velocity component', + 'Uz': 'W velocity component', + } + fieldUnits = {'T': 'K', + 'Ux': 'm s-1', + 'Uy': 'm s-1', + 'Uz': 'm s-1', + } + super().to_netcdf(fname,fieldDescriptions,fieldUnits) + +"""end of class PlanarAverages""" diff --git a/windtools/SOWFA6/postProcessing/probeSets.py b/windtools/SOWFA6/postProcessing/probeSets.py new file mode 100644 index 0000000..a56818c --- /dev/null +++ b/windtools/SOWFA6/postProcessing/probeSets.py @@ -0,0 +1,308 @@ +# Copyright 2020 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +Class for reading in `set` type of OpenFOAM sampling 'probes' + +written by Regis Thedin (regis.thedin@nrel.gov) + +""" +from __future__ import print_function +import os +import pandas as pd +import numpy as np +from .reader import Reader + +class ProbeSets(Reader): + """Stores a time array (t), and field arrays as attributes. The + fields have shape: + (Nt, N[, Nd]) + where N is the number of probes and Nt is the number of samples. + Vectors have an additional dimension to denote vector components. + Symmetric tensors have an additional dimension to denote tensor components (xx, xy, xz, yy, yz, zz). + + The `set`-type of probe is used when large number of data points need to be saved. + Therefore, this class differs from `Probe` and is tailored for the specification of + many sets and looping through the files with ease. The inputs of this class were created + to make it easy to accomodate very large datasets, or only read a subset of the saved data. + + If the need of using `set` arises, chances are the naming of the probes will be complex and likely + inlcude a sweep of a variable in its name. Due to that, the user can specify the name of the probes + split into prefix, suffix, variable sweep, and variables to save. It is also possible to specify a + sub-domain in which data is needed. It is assumed that all sets have the same points. + + Sample usage: + + from windtools.SOWFA6.postProcessing.probeSets import ProbeSets + + # read all times, all variables + probeData = ProbeSet('path/to/case/postProcessing/probeName') + + # read specified fields + probeData = ProbeSet('path/to/case/PostProcessing/probeName', varList['U','T']) + + # read specified sub-domain + probeData = ProbeSet('path/to/case/postProcessing/probeName', xi=-2500, xf=2500, yi=-2500, yf=2500) + + # read all and account for added perturbation on the sampling points + probeData = ProbeSet('path/to/case/postProcessing/probeName', posPert=-0.01) + + # read specified time dirs + probeData = ProbeSet('path/to/case/postProcessing/probeName', tstart=30000, tend=30100) + + # read certain files following complex naming convention + # e.g. if the probes are specified as + ``` + probeName + { + type sets; + name pointcloud; + // other settings... + fields ( U T ); + sets + ( + vmasts_h10 + { + type points; + // ... + } + vmasts_h20 + { + // ... + } + // ... + ) + } + ``` + # and the user wishes to read to vmasts_h{10,50}_{T,U}.xy, then: + probeData = ProbeSet('path/to/case/postProcessing/probeName', + fprefix='vmasts_h', fparam=['10','50'], varList=['T','U'], fsuffix='.xy') + + Notes: + - If `varList` is not specified, then all the probes are read, ignoring prefix, sufix, and parameters + - Pandas/dataframe is used internally even though the final object is of `Reader` type. + + """ + def __init__(self, dpath=None, tstart=None, tend=None, varList='all', posPert=0.0, + xi=None, xf=None, yi=None, yf=None, + fprefix=None, fparam=None, fsuffix=None, + **kwargs): + self.xi = xi + self.xf = xf + self.yi = yi + self.yf = yf + self.fprefix = fprefix + self.fparam = fparam + self.fsuffix = fsuffix + self.posPert = posPert + self.tstart = tstart + self.tend = tend + self.varList = varList + self._allVars = {'U','UMean','T','TMean','TPrimeUPrimeMean','UPrime2Mean','p_rgh'} + super().__init__(dpath,includeDt=True,**kwargs) + + + def _trimtimes(self,tdirList, tstart=None,tend=None): + if (tstart is not None) or (tend is not None): + if tstart is None: tstart = 0.0 + if tend is None: tend = 9e9 + selected = [ (t >= tstart) & (t <= tend) for t in self.times ] + self.filelist = [tdirList[i] for i,b in enumerate(selected) if b ] + self.times = [self.times[i] for i,b in enumerate(selected) if b ] + self.Ntimes = len(self.times) + try: + tdirList = [tdirList[i] for i,b in enumerate(selected) if b ] + except AttributeError: + pass + return tdirList + + + def _processdirs(self, tdirList, trimOverlap=False, **kwargs): + print('Probe data saved:',len(self.simStartTimes), 'time steps, from', \ + self.simStartTimes[0],'s to',self.simStartTimes[-1],'s') + + # make varList iterable if not already a list + varList = [self.varList] if not isinstance(self.varList, (list)) else self.varList + # Create a list of all the probe files that will be processed + if varList[0].lower()=='all': + print('No varList given. Reading all probes.') + outputs = [ fname for fname in os.listdir(tdirList[0]) + if os.path.isfile(tdirList[0]+os.sep+fname) ] + else: + # Make values iterable if not specified list + fprefix = [self.fprefix] if not isinstance(self.fprefix, (list)) else self.fprefix + fparam = [self.fparam] if not isinstance(self.fparam, (list)) else self.fparam + fsuffix = [self.fsuffix] if not isinstance(self.fsuffix, (list)) else self.fsuffix + # create a varList that contains all the files names + fileList = [] + for var in varList: + for prefix in fprefix: + for param in fparam: + for suffix in fsuffix: + fileList.append( prefix + param + '_' + var + suffix ) + outputs = fileList + + # Get list of times and trim the data + self.times = [float(os.path.basename(p)) for p in self.simTimeDirs] + tdirList = self._trimtimes(tdirList,self.tstart,self.tend) + + try: + print('Probe data requested:',len(tdirList), 'time steps, from', \ + float(os.path.basename(tdirList[0])),'s to', \ + float(os.path.basename(tdirList[-1])),'s') + except IndexError: + raise ValueError('End time needs to be greater than the start time') + + # Raise an error if list is empty + if not tdirList: + raise ValueError('No time directories found') + + # Process all data + for field in outputs: + arrays = [ self._read_data( tdir,field ) for tdir in tdirList ] + # combine into a single array and trim end of time series + arrays = np.concatenate(arrays)[:self.imax,:] + # parse the name to create the right variable + param, var = self._parseProbeName(field) + # add the zagl to the array + arrays = np.hstack((arrays[:,:4], \ + np.full((arrays.shape[0],1),param), \ + arrays[:,4:])) + # append to (or create) a variable attribute + try: + setattr(self,var,np.concatenate((getattr(self,var),arrays))) + except AttributeError: + setattr( self, var, arrays ) + + if not var in self._processed: + self._processed.append(var) + print(' read',field) + + self.t = np.unique(arrays[:,0]) + self.Nt = len(self.t) + + # sort times + for var in self._allVars: + try: + self.var = self.var[np.argsort(self.var[:,0])] + except AttributeError: + pass + + + def _parseProbeName(self, field): + # Example: get 'vmasts_50mGrid_h30_T.xy' and return param=30, var='T' + # Remove the prefix from the full field name + f = field.replace(self.fprefix,'') + # Substitude the first underscore with a dot and split array + f = f.replace('_','.',1).split('.') + for i in set(f).intersection(self._allVars): + var = i + param = int(f[-3]) + return param, var + + + def _read_data(self, dpath, fname): + fpath = dpath + os.sep + fname + currentTime = float(os.path.basename(dpath)) + with open(fpath) as f: + try: + # read the actual data from probes + array = self._read_probe_posAndData(f) + # add current time step info to first column + array = np.c_[np.full(array.shape[0],currentTime), array] + except IOError: + print('unable to read '+ fpath) + return array + + + def _read_probe_posAndData(self,f): + out = [] + # Pandas is a LOT faster than reading the file line by line + out = pd.read_csv(f.name,header=None,comment='#',sep='\t') + # Add position perturbation to x, y, zabs + out[[0,1,2]] = out[[0,1,2]].add(self.posPert) + # clip spatial data + out = self._trimpositions(out, self.xi, self.xf, self.yi, self.yf) + out = out.to_numpy(dtype=float) + self.N = len(out) + return out + + + def _trimpositions(self, df, xi=None,xf=None, yi=None, yf=None): + if (xi is not None) and (xf is not None): + df = df.loc[ (df[0]>=xi) & (df[0]<=xf) ] + elif xi is not None: + df = df.loc[ df[0]>=xi ] + elif xf is not None: + df = df.loc[ df[0]<=xf ] + + if (yi is not None) and (yf is not None): + df = df.loc[ (df[1]>=yi) & (df[1]<=yf) ] + elif yi is not None: + df = df.loc[ df[1]>=yi ] + elif yf is not None: + df = df.loc[ df[1]<=yf ] + + return df + + + #============================================================================ + # + # DATA I/O + # + #============================================================================ + + def to_pandas(self,itime=None,fields=None,dtype=None): + #output all vars + if fields is None: + fields = self._processed + # select time range + if itime is None: + tindices = range(len(self.t)) + else: + try: + iter(itime) + except TypeError: + # specified single time index + tindices = [itime] + else: + # specified list of indices + tindices = itime + + # create dataframes for each field + print('Creating dataframe ...') + data = {} + for var in fields: + print('processing', var) + F = getattr(self,var) + # Fill in data + data['time'] = F[:,0] + data['x'] = F[:,1] + data['y'] = F[:,2] + data['zabs'] = F[:,3] + data['zagl'] = F[:,4] + if F.shape[1]==6: + # scalar + data[var] = F[:,5:].flatten() + elif F.shape[1]==8: + # vector + for j,name in enumerate(['x','y','z']): + data[var+name] = F[:,5+j].flatten() + elif F.shape[1]==11: + # symmetric tensor + for j,name in enumerate(['xx','xy','xz','yy','yz','zz']): + data[var+name] = F[:,5+j].flatten() + + df = pd.DataFrame(data=data,dtype=dtype) + return df.sort_values(['time','x','y','zabs','zagl']).set_index(['time','x','y','zagl']) + + def to_netcdf(self,fname,fieldDescriptions={},fieldUnits={}): + raise NotImplementedError('Not available for ProbeSet class.') diff --git a/windtools/SOWFA6/postProcessing/probes.py b/windtools/SOWFA6/postProcessing/probes.py new file mode 100644 index 0000000..0eafade --- /dev/null +++ b/windtools/SOWFA6/postProcessing/probes.py @@ -0,0 +1,138 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +Class for reading in 'probes' type OpenFOAM sampling + +written by Eliot Quon (eliot.quon@nrel.gov) + +""" +from __future__ import print_function +import os +import numpy as np +from .reader import Reader + +def subset_probe(p,indices): + """Return a copy of a Probe object with a subset of locations selected""" + from copy import deepcopy + if isinstance(indices, int): + indices = [indices] + p = deepcopy(p) + p.pos = p.pos[indices,:] + p.N = len(indices) + print('Selected indices:') + print(p.pos) + for field in p._processed: + F = getattr(p,field) + F = F[indices,:] + return p + +class Probe(Reader): + """Stores a time array (t), and field arrays as attributes. The + fields have shape: + (Nt, N[, Nd]) + where N is the number of probes and Nt is the number of samples. + Vectors have an additional dimension to denote vector components. + Symmetric tensors have an additional dimension to denote tensor components (xx, xy, xz, yy, yz, zz). + + Sample usage: + + from windtools.SOWFA6.postProcessing.probes import Probe + + # read all probes + probe = Probe('postProcessing/probe1/') + + # read specified probes only + probe = Probe('postProcessing/probe1/',fields=['U','T']) + + probe.to_csv('probe1.csv') + + """ + def __init__(self,dpath=None,**kwargs): + if 'fields' in kwargs.keys(): + kwargs['varList'] = kwargs.pop('fields') + super().__init__(dpath,**kwargs) + + def _processdirs(self, + tdirList, + varList=['U','T'], + trimOverlap=True + ): + #Redefine _processdirs so that default + #argument for varList can be specified + super()._processdirs(tdirList,varList,trimOverlap) + + def _read_data(self,dpath,fname): + fpath = dpath + os.sep + fname + with open(fpath) as f: + try: + self._read_probe_positions(f) + except IOError: + print('unable to read '+fpath) + else: + array = self._read_probe_data(f) + return array + + + def _read_probe_positions(self,f): + self.pos = [] + line = f.readline() + while '(' in line and ')' in line: + line = line.strip() + assert(line[0]=='#') + assert(line[-1]==')') + iprobe = int(line.split()[2]) + i = line.find('(') + pt = [ float(val) for val in line[i+1:-1].split() ] + self.pos.append( np.array(pt) ) + line = f.readline() + if len(self._processed) > 0: # assert that all fields have same number of probes + assert(self.N == len(self.pos)) + else: # first field: set number of probes in self.N + self.N = len(self.pos) + assert(self.N == iprobe+1) + self.pos = np.array(self.pos) + + + def _read_probe_data(self,f): + line = f.readline() + assert(line.split()[1] == 'Time') + out = [] + for line in f: + line = [ float(val) for val in + line.replace('(','').replace(')','').split() ] + out.append(line) + return np.array(out) + + + #============================================================================ + # + # DATA I/O + # + #============================================================================ + + def to_pandas(self,itime=None,fields=None,dtype=None): + self.hLevelsCell = self.pos[:,2] + return super().to_pandas(itime,fields,dtype) + + def to_netcdf(self,fname): + fieldDescriptions = {'T': 'Potential temperature', + 'Ux': 'U velocity component', + 'Uy': 'V velocity component', + 'Uz': 'W velocity component', + } + fieldUnits = {'T': 'K', + 'Ux': 'm s-1', + 'Uy': 'm s-1', + 'Uz': 'm s-1', + } + self.hLevelsCell = self.pos[:,2] + super().to_netcdf(fname,fieldDescriptions,fieldUnits) diff --git a/windtools/SOWFA6/postProcessing/reader.py b/windtools/SOWFA6/postProcessing/reader.py new file mode 100644 index 0000000..1d08e66 --- /dev/null +++ b/windtools/SOWFA6/postProcessing/reader.py @@ -0,0 +1,351 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +General class for processing SOWFA data + +written by Dries Allaerts (dries.allaerts@nrel.gov) + +""" +from __future__ import print_function +import os +import numpy as np + +class Reader(object): + """Stores a time array (t), and field arrays as attributes. The + fields have shape: + (Nt, N [, Nd]) + Nt is the number of time samples, and N is the number of probes/vertical levels/ ... + Vectors have an additional dimension to denote vector components. + Symmetric tensors have an additional dimension to denote tensor components (xx, xy, xz, yy, yz, zz). + + Sample usage: + + from SOWFA.postProcessing.reader import Reader + + # read all data + data = Reader('postProcessing//') + + # read specified fields only + data = Reader('postProcessing//',varList=['U','T']) + + data.to_csv('data.csv') + + """ + def __init__(self,dpath=None,includeDt=False,**kwargs): + """'Find and process all time directories in path dpath""" + self._processed = [] + self.simTimeDirs = [] #output time names + self.simStartTimes = [] # start or restart simulation times + self.imax = None # for truncating time series + self.Nt = 0 #Number of time samples + self.N = 0 #Number of probes/vertical levels + self.t = None + self.dt = None + self.includeDt = includeDt #Time step is part of output + + if not dpath: + dpath = '.' + + if dpath[-1] == os.sep: dpath = dpath[:-1] # strip trailing slash + + # find results + listing = os.listdir(dpath) + for dirname in listing: + if not os.path.isdir(dpath+os.sep+dirname): continue + try: + startTime = float(dirname) + except ValueError: + # dirname is not a number + pass + else: + self.simTimeDirs.append( dpath+os.sep+dirname ) + self.simStartTimes.append( startTime ) + + if len(self.simTimeDirs) == 0: + # no time directories found; perhaps a single time directory + # was directly specified + dirname = os.path.split(dpath)[-1] + try: + startTime = float(dirname) + except ValueError: + # dirname is not a number + pass + else: + self.simTimeDirs.append( dpath ) + self.simStartTimes.append( startTime ) + + # sort results + self.simTimeDirs = [ x[1] for x in sorted(zip(self.simStartTimes,self.simTimeDirs)) ] + self.simStartTimes.sort() + + # process all output dirs + if len(self.simTimeDirs) > 0: + self._processdirs( self.simTimeDirs, **kwargs ) + else: + print('No time directories found!') + + self._trim_series_if_needed() + + + def _processdirs(self, + tdirList, + varList=[], + trimOverlap=True + ): + """Reads all files within an output time directory. + An object attribute corresponding to the output name + is updated, e.g.: + ${timeDir}/U is appended to the array self.U + """ + print('Simulation (re)start times:',self.simStartTimes) + + if isinstance( varList, (str,) ): + if varList.lower()=='all': + # special case: read all vars + outputs = [ fname for fname in os.listdir(tdirList[0]) + if os.path.isfile(tdirList[0]+os.sep+fname) ] + else: # specified single var + outputs = [varList] + else: #specified list + outputs = varList + + # process all data + selected = [] + for field in outputs: + arrays = [ self._read_data( tdir,field ) for tdir in tdirList ] + + # combine into a single array and trim end of time series + # (because simulations that are still running can have different + # array lengths) + try: + newdata = np.concatenate(arrays)[:self.imax,:] + except ValueError: + print('Could not concatenate the following time-height arrays:') + for tdir,arr in zip(tdirList, arrays): + print(' ', tdir, arr.shape) + + # get rid of overlapped data for restarts + if trimOverlap: + if len(selected) == 0: + # create array mask + tpart = [ array[:,0] for array in arrays ] + for ipart,tcutoff in enumerate(self.simStartTimes[1:]): + selectedpart = np.ones(len(tpart[ipart]),dtype=bool) + try: + iend = np.nonzero(tpart[ipart] >= tcutoff)[0][0] + except IndexError: + # clean restart + pass + else: + # previous simulation didn't finish; overlapped data + selectedpart[iend:] = False + selected.append(selectedpart) + # last / currently running part + selected.append(np.ones(len(tpart[-1]),dtype=bool)) + selected = np.concatenate(selected)[:self.imax] + assert(len(selected) == len(newdata[:,0])) + elif not (len(newdata[:,0]) == len(selected)): + # if simulation is still running, subsequent newdata may + # be longer + self.imax = min(len(selected), len(newdata[:,0])) + selected = selected[:self.imax] + newdata = newdata[:self.imax,:] + # select only unique data + newdata = newdata[selected,:] + + if self.includeDt: + offset = 2 + else: + offset = 1 + # reshape field into (Nt,Nz[,Nd]) and set as attribute + # - note: first column of 'newdata' is time + if newdata.shape[1] == self.N+offset: + # scalar + setattr( self, field, newdata[:,offset:] ) + elif newdata.shape[1] == 3*self.N+offset: + # vector + setattr( self, field, newdata[:,offset:].reshape((newdata.shape[0],self.N,3),order='C') ) + elif newdata.shape[1] == 6*self.N+offset: + # symmetric tensor + setattr( self, field, newdata[:,offset:].reshape((newdata.shape[0],self.N,6),order='C') ) + else: + raise IndexError('Unrecognized number of values') + self._processed.append(field) + print(' read',field) # set time arrays + + self.t = newdata[:,0] + self.Nt = len(self.t) + if self.includeDt: + self.dt = newdata[:,1] + + def _read_data(self,fpath): + return None + + def _trim_series_if_needed(self,fields_to_check=None): + """check for inconsistent array lengths and trim if needed""" + if fields_to_check is None: + fields_to_check = self._processed + for field in fields_to_check: + try: + getattr(self,field) + except AttributeError: + print('Skipping time series length check for unknown field: ', + field) + fields_to_check.remove(field) + field_lengths = [ getattr(self,field).shape[0] for field in fields_to_check ] + if np.min(field_lengths) < np.max(field_lengths): + self.imax = np.min(field_lengths) + # need to prune arrays + print('Inconsistent averaging field lengths... is simulation still running?') + print(' truncated field histories from',np.max(field_lengths),'to',self.imax) + self.t = self.t[:self.imax] + if self.dt is not None: + self.dt = self.dt[:self.imax] + self.Nt = len(self.t) + for field in fields_to_check: + Ndim = len(getattr(self,field).shape) + if Ndim == 2: + # scalar + setattr(self, field, getattr(self,field)[:self.imax,:]) + elif Ndim == 3: + # vector/tensor + setattr(self, field, getattr(self,field)[:self.imax,:,:]) + else: + print('Unknown field type ',field) + + + def __repr__(self): + s = 'Times read: {:d} {:s}\n'.format(self.Nt,str(self.t)) + s+= 'Fields read:\n' + for field in self._processed: + s+= ' {:s} : {:s}\n'.format(field, + str(getattr(self,field).shape)) + return s + + + #============================================================================ + # + # DATA I/O + # + #============================================================================ + + def to_csv(self,fname,**kwargs): + """Write out specified range of times in a pandas dataframe + + kwargs: see Reader.to_pandas() + """ + df = self.to_pandas(**kwargs) + print('Dumping dataframe to',fname) + df.to_csv(fname) + + + def to_pandas(self,itime=None,fields=None,dtype=None): + """Create pandas dataframe for the specified range of times + + Inputs + ------ + itime: integer, list + Time indice(s) to write out; if None, all times are output + fields: list + Name of field variables to write out; if None, all variables + that have been processed are written out + dtype: type + Single datatype to which to cast all fields + """ + import pandas as pd + # output all vars + if fields is None: + fields = self._processed + # select time range + if itime is None: + tindices = range(len(self.t)) + else: + try: + iter(itime) + except TypeError: + # specified single time index + tindices = [itime] + else: + # specified list of indices + tindices = itime + + # create dataframes for each height (with time as secondary index) + # - note: old behavior was to loop over time + # - note: loop over height is much faster when Nt >> Nz + print('Creating dataframe for',self.t[tindices]) + dflist = [] + for i in range(self.N): + data = {} + for var in fields: + F = getattr(self,var) + if len(F.shape)==2: + # scalar + data[var] = F[tindices,i] + elif F.shape[2]==3: + # vector + for j,name in enumerate(['x','y','z']): + data[var+name] = F[tindices,i,j] + elif F.shape[2]==6: + # symmetric tensor + for j,name in enumerate(['xx','xy','xz','yy','yz','zz']): + data[var+name] = F[tindices,i,j] + data['t'] = self.t[tindices] + df = pd.DataFrame(data=data,dtype=dtype) + df['z'] = self.hLevelsCell[i] + dflist.append(df) + return pd.concat(dflist).sort_values(['t','z']).set_index(['t','z']) + + + def to_netcdf(self,fname,fieldDescriptions={},fieldUnits={}): + print('Dumping data to',fname) + import netCDF4 + f = netCDF4.Dataset(fname,'w') + f.createDimension('time',len(self.t)) + f.createDimension('z',len(self.hLevelsCell)) + + times = f.createVariable('time', 'float', ('time',)) + times.long_name = 'Time' + times.units = 's' + times[:] = self.t + + heights = f.createVariable('z', 'float', ('z',)) + heights.long_name = 'Height above ground level' + heights.units = 'm' + heights[:] = self.hLevelsCell + + for var in self._processed: + F = getattr(self,var) + if len(F.shape)==2: + # scalar + varnames = [var,] + F = F[:,:,np.newaxis] + elif F.shape[2]==3: + # vector + varnames = [var+name for name in ['x','y','z']] + elif F.shape[2]==6: + # symmetric tensor + varnames = [var+name for name in ['xx','xy','xz','yy','yz','zz']] + + for i, varname in enumerate(varnames): + field = f.createVariable(varname, 'float', ('time','z')) + try: + field.long_name = fieldDescriptions[varname] + except KeyError: + # Use var name as description + field.long_name = varname + try: + field.units = fieldUnits[varname] + except KeyError: + # Units unknown + pass + field[:] = F[:,:,i] + f.close() diff --git a/windtools/SOWFA6/postProcessing/sourceHistory.py b/windtools/SOWFA6/postProcessing/sourceHistory.py new file mode 100644 index 0000000..f88f85b --- /dev/null +++ b/windtools/SOWFA6/postProcessing/sourceHistory.py @@ -0,0 +1,124 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +For processing SOWFA-6 driving force data in postProcessing/SourceHistory +Based on averaging.py written by Eliot Quon + +written by Dries Allaerts (dries.allaerts@nrel.gov) + +The class can handle both height-dependent and constant source files + +Sample usage: + + from windtools.SOWFA6.postProcessing.sourceHistory import SourceHistory + + # read all time directories in current working directory or in subdirectory called 'SourceHistory' + srcData = SourceHistory() + + # read '0' and '1000' in current working directory + srcData = SourceHistory( 0, 1000 ) + + # read all time directories in specified directory + srcData = SourceHistory('caseX/postProcessing/SourceHistory') + + # read specified time directories + srcData = SourceHistory('caseX/postProcessing/SourceHistory/0', + 'caseX/postProcessing/SourceHistory/1000') +""" +from __future__ import print_function +import os +import numpy as np +import matplotlib.pyplot as plt +from .reader import Reader + + +class SourceHistory(Reader): + + def __init__(self,dpath=None,**kwargs): + super().__init__(dpath,includeDt=True,**kwargs) + + + def _processdirs(self, + tdirList, + varList=['Momentum','Temperature'], + trimOverlap=True + ): + #Redefine _processdirs so that default + #argument for varList can be specified + super()._processdirs(tdirList,varList,trimOverlap) + + + def _read_data(self,dpath,fname): + fpath = dpath + os.sep + 'Source' + fname + 'History' + if fname.startswith('Error'): + fpath = dpath + os.sep + fname + 'History' + + + with open(fpath) as f: + try: + self._read_source_heights(f) + except IOError: + print('unable to read '+fpath) + else: + array = self._read_source_data(f) + return array + + def _read_source_heights(self,f): + line = f.readline().split() + + if line[0].startswith('Time'): + self.hLevelsCell = [0.0] + elif line[0].startswith('Heights'): + self.hLevelsCell = [ float(val) for val in line[2:] ] + f.readline() + else: + print('Error: Expected first line to start with "Time" or "Heights", but instead read',line[0]) + return + + if (len(self._processed) > 0): # assert that all fields have same number of heights + assert (self.N == len(self.hLevelsCell)), \ + 'Error: Various source fields do not have the same number of heights, set varList to read separately' + else: # first field: set number of heights in self.N + self.N = len(self.hLevelsCell) + self.hLevelsCell = np.array(self.hLevelsCell) + return + + + def _read_source_data(self,f): + out = [] + for line in f: + line = [ float(val) for val in + line.replace('(','').replace(')','').split() ] + out.append(line) + return np.array(out) + + + #============================================================================ + # + # DATA I/O + # + #============================================================================ + + def to_netcdf(self,fname): + fieldDescriptions = {'T': 'Potential temperature', + 'Ux': 'U velocity component', + 'Uy': 'V velocity component', + 'Uz': 'W velocity component', + } + fieldUnits = {'T': 'K', + 'Ux': 'm s-1', + 'Uy': 'm s-1', + 'Uz': 'm s-1', + } + super().to_netcdf(fname,fieldDescriptions,fieldUnits) + +"""end of class SourceHistory""" diff --git a/windtools/common.py b/windtools/common.py new file mode 100644 index 0000000..73f0550 --- /dev/null +++ b/windtools/common.py @@ -0,0 +1,190 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import numpy as np +import pandas as pd +import xarray as xr + + +def calc_wind(df=None,u='u',v='v'): + """Calculate wind speed and direction from horizontal velocity + components, u and v. + + Parameters + ========== + df : pd.DataFrame or xr.Dataset + Calculate from data columns (pandas dataframe) or data-arrays + (xarrays dataset) named 'u' and 'v' + u : str or array-like + Data name if 'df' is provided; otherwise array of x-velocities + v : str or array-like + Data name if 'df' is provided; otherwise array of y-velocities + """ + if df is None: + assert (u is not None) and (v is not None) + elif isinstance(df,pd.DataFrame): + assert all(velcomp in df.columns for velcomp in [u,v]), \ + 'velocity components u/v not found; set u and/or v' + u = df[u] + v = df[v] + elif isinstance(df,xr.Dataset): + assert all(velcomp in df.variables for velcomp in [u,v]), \ + 'velocity components u/v not found; set u and/or v' + u = df[u] + v = df[v] + wspd = np.sqrt(u**2 + v**2) + wdir = 180. + np.degrees(np.arctan2(u, v)) + return wspd, wdir + +def calc_uv(df=None,wspd='wspd',wdir='wdir'): + """Calculate velocity components from wind speed and direction. + + Parameters + ========== + df : pd.DataFrame or xr.Dataset + Calculate from data columns (pandas dataframe) or data-arrays + (xarrays dataset) named 'u' and 'v' + wspd : str or array-like + Data name if 'df' is provided; otherwise array of wind speeds + wdir : str or array-like + Data name if 'df' is provided; otherwise array of wind directions + """ + if df is None: + assert (wspd is not None) and (wdir is not None) + elif isinstance(df,pd.DataFrame): + assert all(windcomp in df.columns for windcomp in [wspd,wdir]), \ + 'wind speed/direction not found; set wspd and/or wdir' + wspd = df[wspd] + wdir = df[wdir] + elif isinstance(df,xr.Dataset): + assert all(windcomp in df.variables for windcomp in [wspd,wdir]), \ + 'wind speed/direction not found; set wspd and/or wdir' + wspd = df[wspd] + wdir = df[wdir] + ang = np.radians(270. - wdir) + u = wspd * np.cos(ang) + v = wspd * np.sin(ang) + return u,v + +def fit_powerlaw(df=None,z=None,U=None,zref=80.0,Uref=None): + """Calculate power-law exponent to estimate shear. + + Parameters + ========== + df : pd.DataFrame, optional + Calculate from data columns; index should be height values + U : str or array-like, optional + An array of wind speeds if dataframe 'df' is not provided speeds + z : array-like, optional + An array of heights if dataframe 'df' is not provided + zref : float + Power-law reference height + Uref : float, optional + Power-law reference wind speed; if not specified, then the wind + speeds are evaluatecd at zref to get Uref + + Returns + ======= + alpha : float or pd.Series + Shear exponents + R2 : float or pd.Series + Coefficients of determination + """ + from scipy.optimize import curve_fit + # generalize all inputs + if df is None: + assert (U is not None) and (z is not None) + df = pd.DataFrame(U, index=z) + elif isinstance(df,pd.Series): + df = pd.DataFrame(df) + # make sure we're only working with above-ground values + df = df.loc[df.index > 0] + z = df.index + logz = np.log(z) - np.log(zref) + # evaluate Uref at zref, if needed + if Uref is None: + Uref = df.loc[zref] + elif not hasattr(Uref, '__iter__'): + Uref = pd.Series(Uref,index=df.columns) + # calculate shear coefficient + alpha = pd.Series(index=df.columns) + R2 = pd.Series(index=df.columns) + def fun(x,*popt): + return popt[0]*x + for col,U in df.iteritems(): + logU = np.log(U) - np.log(Uref[col]) + popt, pcov = curve_fit(fun,xdata=logz,ydata=logU,p0=0.14,bounds=(0,1)) + alpha[col] = popt[0] + U = df[col] + resid = U - Uref[col]*(z/zref)**alpha[col] + SSres = np.sum(resid**2) + SStot = np.sum((U - np.mean(U))**2) + R2[col] = 1.0 - (SSres/SStot) + return alpha.squeeze(), R2.squeeze() + + +def covariance(a,b,interval='10min',resample=False,**kwargs): + """Calculate covariance between two series (with datetime index) in + the specified interval, where the interval is defined by a pandas + offset string + (http://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects). + + Notes: + - The output data will have the same length as the input data by + default, because statistics are calculated with pd.rolling(). To + return data at the same intervals as specified, set + `resample=True`. + - Covariances may be simultaneously calculated at multiple heights + by inputting multi-indexed dataframes (with height being the + second index level) + - If the inputs have multiindices, this function will return a + stacked, multi-indexed dataframe. + + Example: + heatflux = covariance(df['Ts'],df['w'],'10min') + """ + # handle xarray data arrays + if isinstance(a, xr.DataArray): + a = a.to_pandas() + if isinstance(b, xr.DataArray): + b = b.to_pandas() + # handle multiindices + have_multiindex = False + if isinstance(a.index, pd.MultiIndex): + assert isinstance(b.index, pd.MultiIndex), \ + 'Both a and b should have multiindices' + assert len(a.index.levels) == 2 + assert len(b.index.levels) == 2 + # assuming levels 0 and 1 are time and height, respectively + a = a.unstack() # create unstacked copy + b = b.unstack() # create unstacked copy + have_multiindex = True + elif isinstance(b.index, pd.MultiIndex): + raise AssertionError('Both a and b should have multiindices') + # check index + if isinstance(interval, str): + # make sure we have a compatible index + assert isinstance(a.index, (pd.DatetimeIndex, pd.TimedeltaIndex, pd.PeriodIndex)) + assert isinstance(b.index, (pd.DatetimeIndex, pd.TimedeltaIndex, pd.PeriodIndex)) + # now, do the calculations + if resample: + a_mean = a.resample(interval).mean() + b_mean = b.resample(interval).mean() + ab_mean = (a*b).resample(interval,**kwargs).mean() + else: + a_mean = a.rolling(interval).mean() + b_mean = b.rolling(interval).mean() + ab_mean = (a*b).rolling(interval,**kwargs).mean() + cov = ab_mean - a_mean*b_mean + if have_multiindex: + return cov.stack() + else: + return cov diff --git a/windtools/inflow/general.py b/windtools/inflow/general.py new file mode 100644 index 0000000..18a0351 --- /dev/null +++ b/windtools/inflow/general.py @@ -0,0 +1,517 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import sys,os +import time +import numpy as np + +import windtools.SOWFA6.constant.boundaryData as bc +from windtools.io.vtk import vtk_write_structured_points + + +class InflowPlane(object): + """This is the base class for all inflows. User should create an + instance of one of the derived classes in the synthetic module. + """ + + realtype = np.float32 + + def __init__(self, verbose=False): + """Base initializer for inflow data, which should be overridden. + Defaults are set here. + + After initialization, the following variables should be set: + * Dimensions: NY, NZ (horizontal, vertical) + * Number of time snapshots: N + * Spacings/step size: dt or dx, dy, dz + * Rectilinear grid: y, z + * Sampling times or streamwisei coordinate: t or x + * Velocity field: U (with shape==(3,Ntimes,NY,NZ)) + * Potential temperature field: T (with shape==(Ntimes,NY,NZ)) + * Scaling function: scaling (shape==(3,NZ)) + + Optionally, the following parameters may be set: + * Reference velocity: Umean + """ + self.verbose = verbose + self.Umean = None # reference velocity + self.have_field = False # True after the velocity field has been read + +# self.needUpdateMean = False # set to true update the mean inflow at every time step. +# self.timeseries = None # used if needUpdateMean is True +# self.Useries = None # used if needUpdateMean is True +# self.Tseries = None # used if needUpdateMean is True + + self.mean_flow_read = False + self.variances_read = False + + # inflow plane coordinates + self.y = None + self.z = None + + # set by calcVariance + self.uu_mean = None + self.vv_mean = None + self.ww_mean = None + + # constant profiles, set by readAllProfiles or readVarianceProfile) + self.z_profile = None + self.uu_profile = None + self.vv_profile = None + self.ww_profile = None + + + def read_field(self): + print('This is a function stub; no inflow data were read.') + + + def calcVariance(self,output=None): + """Calculate variance of the fluctuating velocities. The square + root (i.e., the standard deviation) should match the output in + PREFIX.sum. + """ + self.uu = self.U[0,:,:,:]**2 + self.vv = self.U[1,:,:,:]**2 + self.ww = self.U[2,:,:,:]**2 + self.uu_tavg = np.mean(self.uu,0) # time averages + self.vv_tavg = np.mean(self.vv,0) + self.ww_tavg = np.mean(self.ww,0) + self.uu_mean = np.mean( self.uu_tavg ) # space/time average + self.vv_mean = np.mean( self.vv_tavg ) + self.ww_mean = np.mean( self.ww_tavg ) + + print('Spatial average of , , :',self.uu_mean,self.vv_mean,self.ww_mean) + + if output is not None: + with open(output,'w') as f: + f.write('Spatial average of , , : {} {} {}\n'.format(self.uu_mean,self.vv_mean,self.ww_mean)) + f.write('\n Height Standard deviation at grid points for the u component:\n') + for i,zi in enumerate(self.z): + f.write('z= {:.1f} : {}\n'.format(zi,np.sqrt(self.uu_tavg[:,i]))) + f.write('\n Height Standard deviation at grid points for the v component:\n') + for i,zi in enumerate(self.z): + f.write('z= {:.1f} : {}\n'.format(zi,np.sqrt(self.vv_tavg[:,i]))) + f.write('\n Height Standard deviation at grid points for the w component:\n') + for i,zi in enumerate(self.z): + f.write('z= {:.1f} : {}\n'.format(zi,np.sqrt(self.ww_tavg[:,i]))) + print('Wrote out',output) + + + # + # Domain manipulation functions + # + + def tileY(self,ntiles,mirror=False): + """Duplicate field in lateral direction + 'ntiles' is the final number of panels including the original + + Set 'mirror' to True to flip every other tile + """ + ntiles = int(ntiles) + print('Creating',ntiles,'horizontal tiles') + print(' before:',self.U.shape) + if mirror: + # [0 1 2] --> [0 1 2 1 0 1 2 .. ] + NYnew = (self.NY-1)*ntiles + 1 + Unew = np.zeros((3,self.N,NYnew,self.NZ)) + Tnew = np.zeros(( self.N,NYnew,self.NZ)) + Unew[:,:,:self.NY,:] = self.U[:,:,:self.NY,:] + Tnew[ :,:self.NY,:] = self.T[ :,:self.NY,:] + delta = self.NY - 1 + flipped = True + for i in range(1,ntiles): + if flipped: + Unew[:,:,i*delta+1:(i+1)*delta+1,:] = self.U[:,:,delta-1::-1,:] + Tnew[ :,i*delta+1:(i+1)*delta+1,:] = self.T[ :,delta-1::-1,:] + else: + Unew[:,:,i*delta+1:(i+1)*delta+1,:] = self.U[:,:,1:,:] + Tnew[ :,i*delta+1:(i+1)*delta+1,:] = self.T[ :,1:,:] + flipped = not flipped + self.U = Unew + self.T = Tnew + else: + # [0 1 2] --> [0 1 0 1 .. 0 1 2] + self.U = np.tile(self.U[:,:,:-1,:],(1,1,ntiles,1)) + self.T = np.tile(self.T[ :,:-1,:],( 1,ntiles,1)) + Uplane0 = np.zeros((3,self.N,1,self.NZ)) + Tplane0 = np.zeros(( self.N,1,self.NZ)) + Uplane0[:,:,0,:] = self.U[:,:,-1,:] + Tplane0[ :,0,:] = self.T[ :,-1,:] + self.U = np.concatenate((self.U,Uplane0),axis=1) + self.T = np.concatenate((self.T,Tplane0),axis=1) + print(' after :',self.U.shape) + + self.NY = NYnew + assert( self.U.shape == (3,self.N,self.NY,self.NZ) ) + self.y = np.arange(self.NY,dtype=self.realtype)*self.dy + + + def resizeY(self,yMin=None,yMax=None,dryrun=False): + """Resize inflow domain to fit LES boundary and update NY. + Min(y) will be shifted to coincide with yMin. + """ + if yMin is None: + yMin = self.y[0] + if yMax is None: + yMax = self.y[-1] + Ly_specified = yMax - yMin + Ly = self.y[-1] - self.y[0] + if Ly_specified > Ly: + print('Specified y range', (yMin,yMax), + 'greater than', (self.y[0],self.y[-1])) + return + + if dryrun: sys.stdout.write('(DRY RUN) ') + print('Resizing fluctuations field in y-dir from [', + self.y[0],self.y[-1],'] to [',yMin,yMax,']') + print(' before:',self.U.shape) + + newNY = int(np.ceil(Ly_specified/Ly * self.NY)) + Unew = self.U[:,:,:newNY,:] + Tnew = self.T[ :,:newNY,:] + print(' after:',Unew.shape) + if not dryrun: + self.U = Unew + self.T = Tnew + self.NY = newNY + + ynew = yMin + np.arange(newNY,dtype=self.realtype)*self.dy + if not dryrun: + print('Updating y coordinates') + self.y = ynew + else: + print('(DRY RUN) y coordinates:',ynew) + + + def resizeZ(self,zMin=None,zMax=None,shrink=False,dryrun=False): + """Set/extend inflow domain to fit LES boundary and update NZ. + Values between zMin and min(z) will be duplicated from + V[:3,y,z=min(z),t], whereas values between max(z) and zMax will + be set to zero. + + By default, this function will not resize inflow plane to a + smaller domain; to override this, set shrink to True. + """ + if zMin is None: + zMin = self.z[0] + if zMax is None: + zMax = self.z[-1] + if not shrink: + if zMin > self.z[0]: + print('zMin not changed from',self.z[0],'to',zMin) + return + if zMax < self.z[-1]: + print('zMax not changed from',self.z[-1],'to',zMax) + return + + self.zbot = zMin + + imin = int((zMin-self.z[0])/self.dz) + imax = int(np.ceil((zMax-self.z[0])/self.dz)) + zMin = imin*self.dz + self.z[0] + zMax = imax*self.dz + self.z[0] + ioff = int((self.z[0]-zMin)/self.dz) + if dryrun: sys.stdout.write('(DRY RUN) ') + print('Resizing fluctuations field in z-dir from [', + self.z[0],self.z[-1],'] to [',zMin,zMax,']') + print(' before:',self.U.shape) + + newNZ = imax-imin+1 + Unew = np.zeros((3,self.N,self.NY,newNZ)) + Tnew = np.zeros(( self.N,self.NY,newNZ)) + for iz in range(ioff): + Unew[:,:,:,iz] = self.U[:,:,:,0] + Tnew[ :,:,iz] = self.T[ :,:,0] + if not shrink: + Unew[:,:,:,ioff:ioff+self.NZ] = self.U + Tnew[ :,:,ioff:ioff+self.NZ] = self.T + else: + iupper = np.min((ioff+self.NZ, newNZ)) + Unew[:,:,:,ioff:iupper] = self.U[:,:,:,:iupper-ioff] + Tnew[ :,:,ioff:iupper] = self.T[ :,:,:iupper-ioff] + print(' after:',Unew.shape) + if not dryrun: + self.U = Unew + self.T = Tnew + self.NZ = newNZ + + znew = self.zbot + np.arange(newNZ,dtype=self.realtype)*self.dz + if not dryrun: + print('Updating z coordinates') + self.z = znew + else: + print('(DRY RUN) z coordinates:',znew) + + if not dryrun: + print('Resetting scaling function') + self.scaling = np.ones((3,newNZ)) + + + # + # Boundary output + # + + def write_sowfa_mapped_BC(self, + outputdir='boundaryData', + time_varying_input=None, + ref_height=None, + bcname='west', + xinlet=0.0, + tstart=0.0, + periodic=False): + """For use with OpenFOAM's timeVaryingMappedFixedValue boundary + condition. This will create a points file and time directories + in 'outputdir', which should be placed in + constant/boundaryData/. + + time_varying_input should be a dictionary of (NT, NY, NZ, 3) + arrays which shoud be aligned with the loaded data in terms of + (dy, dz, dt, and NT) + """ + dpath = os.path.join(outputdir, bcname) + if not os.path.isdir(dpath): + print('Creating output dir :',dpath) + os.makedirs(dpath) + + if ref_height is not None: assert(self.z is not None) + + # TODO: check time-varying input + assert(time_varying_input is not None) + Uinput = time_varying_input['U'] + Tinput = time_varying_input['T'] + kinput = time_varying_input['k'] + NT, NY, NZ, _ = Uinput.shape + u = np.zeros((NY,NZ)) # working array + v = np.zeros((NY,NZ)) # working array + w = np.zeros((NY,NZ)) # working array + T = np.zeros((NY,NZ)) # working array + + # write points + fname = os.path.join(dpath,'points') + print('Writing',fname) + with open(fname,'w') as f: + f.write(bc.pointsheader.format(patchName=bcname,N=NY*NZ)) + for k in range(NZ): + for j in range(NY): + f.write('({:f} {:f} {:f})\n'.format(xinlet, + self.y[j], + self.z[k])) + f.write(')\n') + + # begin time-step loop + for itime in range(NT): + curtime = self.realtype(tstart + (itime+1)*self.dt) + tname = '{:f}'.format(curtime).rstrip('0').rstrip('.') + + prefix = os.path.join(dpath,tname) + if not os.path.isdir(prefix): + os.makedirs(prefix) + + # get fluctuations at current time + if periodic: + itime0 = np.mod(itime, self.N) + else: + itime0 = itime + #u[:,:] = self.U[0,itime0,:NY,:NZ] # self.U.shape==(3, NT, NY, NZ) + #v[:,:] = self.U[1,itime0,:NY,:NZ] # self.U.shape==(3, NT, NY, NZ) + utmp = self.U[0,itime0,:NY,:NZ].copy() + vtmp = self.U[1,itime0,:NY,:NZ].copy() + w[:,:] = self.U[2,itime0,:NY,:NZ] # self.U.shape==(3, NT, NY, NZ) + T[:,:] = self.T[itime0,:NY,:NZ] # self.T.shape==(NT, NY, NZ) + + # scale fluctuations + for iz in range(NZ): # note: u is the original size + utmp[:,iz] *= self.scaling[0,iz] + vtmp[:,iz] *= self.scaling[1,iz] + w[:,iz] *= self.scaling[2,iz] + + # rotate fluctuating field + # TODO: allow for constant input + winddir_profile = np.mean(np.arctan2(Uinput[itime,:,:,1], + Uinput[itime,:,:,0]), axis=0) + if ref_height is not None: + mean_winddir = np.interp(ref_height, self.z, winddir_profile) + else: + mean_winddir = np.mean(winddir_profile) + mean_winddir_compass = 270.0 - 180.0/np.pi*mean_winddir + if mean_winddir_compass < 0: + mean_winddir_compass += 360.0 + if ref_height is not None: + print(('Mean wind dir at {:.1f} m is {:.1f} deg,' \ + + ' rotating by {:.1f} deg').format(ref_height, + mean_winddir_compass, mean_winddir*180.0/np.pi)) + else: + print(('Mean wind dir is {:.1f} deg,' \ + + ' rotating by {:.1f} deg').format( + mean_winddir_compass, mean_winddir*180.0/np.pi)) + u[:,:] = utmp*np.cos(mean_winddir) - vtmp*np.sin(mean_winddir) + v[:,:] = utmp*np.sin(mean_winddir) + vtmp*np.cos(mean_winddir) + + # superimpose inlet snapshot + u[:,:] += Uinput[itime,:,:,0] + v[:,:] += Uinput[itime,:,:,1] + w[:,:] += Uinput[itime,:,:,2] + T[:,:] += Tinput[itime,:,:] + + # write out U + fname = os.path.join(prefix,'U') + print('Writing out',fname) + bc.write_data(fname, + np.stack((u.ravel(order='F'), + v.ravel(order='F'), + w.ravel(order='F'))), + patchName=bcname, + timeName=tname, + avgValue=[0,0,0]) + + # write out T + fname = os.path.join(prefix,'T') + print('Writing out',fname) + bc.write_data(fname, + T.ravel(order='F'), + patchName=bcname, + timeName=tname, + avgValue=0) + + # write out k + fname = os.path.join(prefix,'k') + print('Writing out',fname) + bc.write_data(fname, + kinput[itime,:,:].ravel(order='F'), + patchName=bcname, + timeName=tname, + avgValue=0) + + + # + # Visualization output + # + + def writeVTK(self, fname, + itime=None, + output_time=None, + scaled=True, + stdout='overwrite'): + """Write out binary VTK file with a single vector field for a + specified time index or output time. + """ + if output_time: + itime = int(output_time / self.dt) + if itime is None: + print('Need to specify itime or output_time') + return + if stdout=='overwrite': + sys.stdout.write('\rWriting time step {:d} : t= {:f}'.format( + itime,self.t[itime])) + else: #if stdout=='verbose': + print('Writing out VTK for time step',itime,': t=',self.t[itime]) + + # scale fluctuations + up = np.zeros((1,self.NY,self.NZ)) # constant x plane (3D array for VTK output) + wp = np.zeros((1,self.NY,self.NZ)) + vp = np.zeros((1,self.NY,self.NZ)) + up[0,:,:] = self.U[0,itime,:,:] + vp[0,:,:] = self.U[1,itime,:,:] + wp[0,:,:] = self.U[2,itime,:,:] + if scaled: + for iz in range(self.NZ): + up[0,:,iz] *= self.scaling[0,iz] + vp[0,:,iz] *= self.scaling[1,iz] + wp[0,:,iz] *= self.scaling[2,iz] + + # calculate instantaneous velocity + U = up.copy() + V = vp.copy() + W = wp.copy() + if self.mean_flow_read: + for iz in range(self.NZ): + U[0,:,iz] += self.U_inlet[:,iz] + V[0,:,iz] += self.V_inlet[:,iz] + W[0,:,iz] += self.W_inlet[:,iz] + + # write out VTK + vtk_write_structured_points( + open(fname,'wb'), #binary mode + { + "U": np.stack((U,V,W)), + "u'": np.stack((up,vp,wp)), + }, + dx=1.0, dy=self.dy, dz=self.dz, + origin=[0.,self.y[0],self.z[0]], + indexorder='ijk', + ) + + + def writeVTKSeries(self, + outputdir='.', + prefix='inflow', + step=1, + scaled=True, + stdout='overwrite'): + """Driver for writeVTK to output a range of times""" + if not os.path.isdir(outputdir): + print('Creating output dir :',outputdir) + os.makedirs(outputdir) + + for i in range(0,self.N,step): + fname = os.path.join(outputdir, f'{prefix:s}_{i:06d}.vtk') + self.writeVTK(fname,itime=i,scaled=scaled,stdout=stdout) + if stdout=='overwrite': sys.stdout.write('\n') + + + def writeVTKBlock(self, + fname='turbulence_box.vtk', + outputdir=None, + step=1, + scaled=True): + """Write out a 3D block wherein the x planes are comprised of + temporal snapshots spaced (Umean * step * dt) apart. + + This invokes Taylor's frozen turbulence assumption. + """ + if outputdir is None: + outputdir = '.' + elif not os.path.isdir(outputdir): + print('Creating output dir :',outputdir) + os.makedirs(outputdir) + + fname = os.path.join(outputdir,fname) + print('Writing VTK block',fname) + + if self.Umean is not None: + Umean = self.Umean + else: + Umean = 1.0 + + # scale fluctuations + Nt = int(self.N / step) + up = np.zeros((Nt,self.NY,self.NZ)) + vp = np.zeros((Nt,self.NY,self.NZ)) + wp = np.zeros((Nt,self.NY,self.NZ)) + up[:,:,:] = self.U[0,:Nt*step:step,:,:] + vp[:,:,:] = self.U[1,:Nt*step:step,:,:] + wp[:,:,:] = self.U[2,:Nt*step:step,:,:] + if scaled: + for iz in range(self.NZ): + up[:,:,iz] *= self.scaling[0,iz] + vp[:,:,iz] *= self.scaling[1,iz] + wp[:,:,iz] *= self.scaling[2,iz] + + # write out VTK + vtk_write_structured_points( open(fname,'wb'), #binary mode + Nt, self.NY, self.NZ, + [ up,vp,wp ], + datatype=['vector'], + dx=step*Umean*self.dt, dy=self.dy, dz=self.dz, + dataname=['u\''], + origin=[0.,self.y[0],self.z[0]], + indexorder='ijk') + diff --git a/windtools/inflow/synthetic.py b/windtools/inflow/synthetic.py new file mode 100644 index 0000000..8cce803 --- /dev/null +++ b/windtools/inflow/synthetic.py @@ -0,0 +1,271 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import sys,os +import time +import numpy as np + +from .general import InflowPlane +from windtools.io.binary import BinaryFile + + +class TurbSim(InflowPlane): + + def __init__(self, fname=None, Umean=None, verbose=False, **kwargs): + """Processes binary full-field time series output from TurbSim. + + Tested with TurbSim v2.00.05c-bjj, 25-Feb-2016 + Tested with pyTurbsim, 10-07-2017 + """ + super(self.__class__,self).__init__(verbose,**kwargs) + self.Umean = Umean + + if fname is not None: + self.read_field(fname) + + + def read_field(self,fname): + if not fname.endswith('.bts'): + fname = fname + '.bts' + self._readBTS(fname) + self.have_field = True + + def _readBTS(self,fname): + """ Process AeroDyn full-field files. Fluctuating velocities and + coordinates (y & z) are calculated. + + V.shape = (3,NY,NZ,N) # N: number of time steps + """ + with BinaryFile(fname) as f: + # + # read header info + # + if self.verbose: print('Reading header information from',fname) + + ID = f.read_int2() + assert( ID==7 or ID==8 ) + if ID==7: filetype = 'non-periodic' + elif ID==8: filetype = 'periodic' + else: filetype = 'UNKNOWN' + if self.verbose: + print(' id= {:d} ({:s})'.format(ID,filetype)) + + # - read resolution settings + self.NZ = f.read_int4() + self.NY = f.read_int4() + self.Ntower = f.read_int4() + if self.verbose: + print(' NumGrid_Z,_Y=',self.NZ,self.NY) + print(' ntower=',self.Ntower) + self.N = f.read_int4() + self.dz = f.read_float(dtype=self.realtype) + self.dy = f.read_float(dtype=self.realtype) + self.dt = f.read_float(dtype=self.realtype) + self.period = self.realtype(self.N * self.dt) + self.Nsize = 3*self.NY*self.NZ*self.N + if self.verbose: + print(' nt=',self.N) + print(' (problem size: {:d} points)'.format(self.Nsize)) + print(' dz,dy=',self.dz,self.dy) + print(' TimeStep=',self.dt) + print(' Period=',self.period) + + # - read reference values + self.uhub = f.read_float(dtype=self.realtype) + self.zhub = f.read_float(dtype=self.realtype) # NOT USED + self.zbot = f.read_float(dtype=self.realtype) + if self.Umean is None: + self.Umean = self.uhub + if self.verbose: + print(' Umean = uhub =',self.Umean, + '(for calculating fluctuations)') + else: # user-specified Umean + if self.verbose: + print(' Umean =',self.Umean, + '(for calculating fluctuations)') + print(' uhub=',self.uhub,' (NOT USED)') + if self.verbose: + print(' HubHt=',self.zhub,' (NOT USED)') + print(' Zbottom=',self.zbot) + + # - read scaling factors + self.Vslope = np.zeros(3,dtype=self.realtype) + self.Vintercept = np.zeros(3,dtype=self.realtype) + for i in range(3): + self.Vslope[i] = f.read_float(dtype=self.realtype) + self.Vintercept[i] = f.read_float(dtype=self.realtype) + if self.verbose: + # output is float64 precision by default... + print(' Vslope=',self.Vslope) + print(' Vintercept=',self.Vintercept) + + # - read turbsim info string + nchar = f.read_int4() + version = f.read(N=nchar) + if self.verbose: print(version) + + # + # read normalized data + # + # note: need to specify Fortran-order to properly read data using np.nditer + t0 = time.process_time() + if self.verbose: print('Reading normalized grid data') + + self.U = np.zeros((3,self.NY,self.NZ,self.N),order='F',dtype=self.realtype) + self.T = np.zeros((self.N,self.NY,self.NZ)) + if self.verbose: + print(' U size :',self.U.nbytes/1024.**2,'MB') + + for val in np.nditer(self.U, op_flags=['writeonly']): + val[...] = f.read_int2() + self.U = self.U.swapaxes(3,2).swapaxes(2,1) # new shape: (3,self.N,self.NY,self.NZ) + + if self.Ntower > 0: + if self.verbose: + print('Reading normalized tower data') + self.Utow = np.zeros((3,self.Ntower,self.N), + order='F',dtype=self.realtype) + if self.verbose: + print(' Utow size :',self.Utow.nbytes/1024.**2,'MB') + for val in np.nditer(self.Utow, op_flags=['writeonly']): + val[...] = f.read_int2() + + if self.verbose: + print(' Read velocitiy fields in',time.process_time()-t0,'s') + + # + # calculate dimensional velocity + # + if self.verbose: + print('Calculating velocities from normalized data') + for i in range(3): + self.U[i,:,:,:] -= self.Vintercept[i] + self.U[i,:,:,:] /= self.Vslope[i] + if self.Ntower > 0: + self.Utow[i,:,:] -= self.Vintercept[i] + self.Utow[i,:,:] /= self.Vslope[i] + self.U[0,:,:,:] -= self.Umean # uniform inflow w/ no shear assumed + + print(' u min/max [',np.min(self.U[0,:,:,:]), + np.max(self.U[0,:,:,:]),']') + print(' v min/max [',np.min(self.U[1,:,:,:]), + np.max(self.U[1,:,:,:]),']') + print(' w min/max [',np.min(self.U[2,:,:,:]), + np.max(self.U[2,:,:,:]),']') + + self.scaling = np.ones((3,self.NZ)) + + # + # calculate coordinates + # + if self.verbose: + print('Calculating coordinates') + #self.y = -0.5*(self.NY-1)*self.dy + np.arange(self.NY,dtype=self.realtype)*self.dy + self.y = np.arange(self.NY,dtype=self.realtype)*self.dy + self.z = self.zbot + np.arange(self.NZ,dtype=self.realtype)*self.dz + #self.ztow = self.zbot - np.arange(self.NZ,dtype=self.realtype)*self.dz #--NOT USED + + self.t = np.arange(self.N,dtype=self.realtype)*self.dt + if self.verbose: + print('Read times [',self.t[0],self.t[1],'...',self.t[-1],']') + + +class GaborKS(InflowPlane): + + def __init__(self, prefix=None, + tidx=0, + dt=None, Umean=None, + potentialTemperature=None, + verbose=True, + **kwargs): + """Processes binary output from Gabor KS. + """ + super(self.__class__,self).__init__(verbose,**kwargs) + + fieldnames = ['uVel','vVel','wVel'] + self.Ncomp = 3 + if potentialTemperature is not None: + self.Ncomp += 1 + print('Note: Potential temperature is not currently handled!') + fieldnames.append('potT') + + self.fnames = [ '{}_{}_t{:06d}.out'.format(prefix,fieldvar,tidx) for fieldvar in fieldnames ] + self.infofile = '{}_info_t{:06d}.out'.format(prefix,tidx) + self.Umean = Umean + self.dt = dt + + self.read_info(self.infofile) + + if self.dt is None and self.Umean is None: + self.dt = 1.0 + self.Umean = self.dx + elif self.Umean is None: + self.Umean = self.dx / self.dt + print('Specified dt =',self.dt) + print('Calculated Umean =',self.Umean) + elif self.dt is None: + self.dt = self.dx / self.Umean + print('Specified Umean =',self.Umean) + print('Calculated dt =',self.dt) + else: + if self.verbose: + print('Specified Umean, dt =',self.Umean,self.dt) + + self.t = np.arange(self.NX)*self.dt + self.y = np.arange(self.NY)*self.dy + self.z = np.arange(self.NZ)*self.dz + if self.verbose: + print('t range:',[np.min(self.t),np.max(self.t)]) + print('y range:',[np.min(self.y),np.max(self.y)]) + print('z range:',[np.min(self.z),np.max(self.z)]) + + if self.fnames is not None: + self.read_field(self.fnames) + + + def read_info(self,fname): + info = np.genfromtxt(fname, dtype=None) + self.t0 = info[0] + self.NX = int(info[1]) + self.NY = int(info[2]) + self.NZ = int(info[3]) + self.Lx = info[4] + self.Ly = info[5] + self.Lz = info[6] + self.N = self.NX # time steps equal to x planes + self.dx = self.Lx/self.NX + self.dy = self.Ly/self.NY + self.dz = self.Lz/self.NZ + + self.xG,self.yG,self.zG = np.meshgrid( + np.linspace(0,self.Lx-self.dx,self.NX), + np.linspace(0,self.Ly-self.dy,self.NY), + np.linspace(self.dz/2,self.Lz-(self.dz/2),self.NZ), + indexing='ij') + + print('Read info file',fname) + if self.verbose: + print(' domain dimensions:',[self.NX,self.NY,self.NZ]) + print(' domain extents:',[self.Lx,self.Ly,self.Lz],'m') + + + def read_field(self,fnames): + self.U = np.zeros((self.Ncomp,self.NX,self.NY,self.NZ)) + self.T = np.zeros((self.NX,self.NY,self.NZ)) + self.scaling = np.ones((3,self.NZ)) + + for icomp,fname in enumerate(self.fnames): + tmpdata = np.fromfile(fname,dtype=np.dtype(np.float64),count=-1) + self.U[icomp,:,:,:] = tmpdata.reshape((self.NX,self.NY,self.NZ),order='F') + + self.have_field = True + + diff --git a/windtools/io/binary.py b/windtools/io/binary.py new file mode 100644 index 0000000..c001295 --- /dev/null +++ b/windtools/io/binary.py @@ -0,0 +1,122 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import numpy as np +import struct + + +class BinaryFile: + """ + Helper class for handling binary file I/O + """ + def __init__(self,path,mode='rb'): + """Create binary file reader object + + Sample Usage + ------------ + ``` + from windtools.io.binary import BinaryFile + with BinaryFile(fname) as f: + i = f.read_int4() + f = f.read_float() + someArray = f.read_float(10) + ``` + """ + self.path = path + self.mode = mode.strip('b') + self.f = open(path,self.mode+'b') + + def __enter__(self): + # Called when used with the 'with' statement + return self + + def __exit__(self, exc_type, exc_value, traceback): + """ Cleandown code, called when used with the 'with' statement + ref: http://stackoverflow.com/questions/22417323/how-do-enter-and-exit-work-in-python-decorator-classes + """ + # clean up + self.f.close() + # handle exceptions + if exc_type is not None: + print(exc_type, exc_value, traceback) + #return False # uncomment to pass exception through + return self + + def close(self): + self.f.close() + + def read(self,N=1): + return self.f.read(N) + + def unpack(self,*args): + try: + return struct.unpack(*args) + except struct.error: + raise IOError + + def read_char(self): + return self.f.read(1).decode('utf-8') + + def readline(self): + s = '' + b = self.read_char() + while not b == '\n': + s += b + b = self.f.read(1).decode('utf-8') + return s + b + + # integers + def read_int1(self,N=1): + if N==1: return self.unpack('b',self.f.read(1))[0] #short + else: return self.unpack('{:d}b',self.f.read(N*1))[0:N] #short + def read_int2(self,N=1): + if N==1: return self.unpack('h',self.f.read(2))[0] #short + else: return self.unpack('{:d}h'.format(N),self.f.read(N*2))[0:N] #short + def read_int4(self,N=1): + if N==1: return self.unpack('i',self.f.read(4))[0] #int + else: return self.unpack('{:d}i'.format(N),self.f.read(N*4))[0:N] #int + def read_int8(self,N=1): + if N==1: return self.unpack('l',self.f.read(8))[0] #long + else: return self.unpack('{:d}l'.format(N),self.f.read(N*8))[0:N] #long + + # floats + def read_float(self,N=1,dtype=float): + if N==1: return dtype( self.unpack('f',self.f.read(4))[0] ) + else: return [ dtype(val) for val in self.unpack('{:d}f'.format(N),self.f.read(N*4))[0:N] ] + def read_double(self,N=1): + if N==1: return self.unpack('d',self.f.read(8))[0] + else: return self.unpack('{:d}d'.format(N),self.f.read(N*8))[0:N] + def read_real4(self,N=1): + return self.read_float(N,dtype=np.float32) + def read_real8(self,N=1): + return self.read_float(N,dtype=np.float64) + + # binary output + def write_type(self,val,type): + if hasattr(val,'__iter__'): + N = len(val) + self.f.write(struct.pack('{:d}{:s}'.format(N,type),*val)) + else: + self.f.write(struct.pack(type,val)) + + + # aliases + def read_int(self,N=1): return self.read_int4(N) + + def write_int1(self,val): self.write_type(val,'b') + def write_int2(self,val): self.write_type(val,'h') + def write_int4(self,val): self.write_type(val,'i') + def write_int8(self,val): self.write_type(val,'l') + + def write_int(self,val): self.write_int4(val) + def write_float(self,val): self.write_type(val,'f') + def write_double(self,val): self.write_type(val,'d') + diff --git a/windtools/io/ensight.py b/windtools/io/ensight.py new file mode 100644 index 0000000..f5faa0d --- /dev/null +++ b/windtools/io/ensight.py @@ -0,0 +1,47 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import os +import pandas as pd + + +def read_mesh(fpath,headerlength=8,chunksize=None): + """Read Ensight mesh file (ascii) into a dataframe""" + with open(fpath,'r') as f: + for _ in range(headerlength): + f.readline() + N = int(f.readline()) + if chunksize is None: + mesh = pd.read_csv(f,header=None,nrows=3*N).values + else: + mesh = pd.concat(pd.read_csv(f,header=None,nrows=3*N,chunksize=chunksize)).values + df = pd.DataFrame(data=mesh.reshape((N,3),order='F'), columns=['x','y','z']) + return df + + +def read_vector(fpath,mesh,t=0.0,headerlength=4,chunksize=None): + """Read Ensight data array (ascii) into a dataframe with combined mesh + information corresponding to the specified time; mesh should be read in by + the read_mesh() function + """ + Npts = len(mesh) + with open(fpath,'r') as f: + for _ in range(headerlength): + f.readline() + if chunksize is None: + vals = pd.read_csv(f,header=None,nrows=3*Npts).values + else: + vals = pd.concat(pd.read_csv(f,header=None,nrows=3*Npts,chunksize=chunksize)).values + df = mesh.copy() + df['t'] = t + uvw = pd.DataFrame(data=vals.reshape((Npts,3),order='F'), columns=['u','v','w']) + df = pd.concat([df,uvw], axis=1) + return df.set_index(['t','x','y','z']) diff --git a/windtools/io/series.py b/windtools/io/series.py new file mode 100644 index 0000000..9f262a6 --- /dev/null +++ b/windtools/io/series.py @@ -0,0 +1,232 @@ +""" +Classes for organizing data series stored in different directory and +subdirectory structures +""" +import os + +def pretty_list(strlist,indent=2,sep='\t',width=80): + """For formatting long lists of strings of arbitrary length + """ + sep = sep.expandtabs() + max_item_len = max([len(s) for s in strlist]) + items_per_line = int((width - (indent+max_item_len)) / (len(sep)+max_item_len) + 1) + Nlines = int(len(strlist) / items_per_line) + extraline = (len(strlist) % items_per_line) > 0 + fmtstr = '{{:{:d}s}}'.format(max_item_len) + strlist = [ fmtstr.format(s) for s in strlist ] # pad strings so that they're all the same length + finalline = '' + for line in range(Nlines): + ist = line*items_per_line + finalline += indent*' ' + sep.join(strlist[ist:ist+items_per_line]) + '\n' + if extraline: + finalline += indent*' ' + sep.join(strlist[Nlines*items_per_line:]) + '\n' + return finalline + + +class Series(object): + """Object for holding general series data + + Written by Eliot Quon (eliot.quon@nrel.gov) + """ + def __init__(self,datadir,**kwargs): + self.datadir = os.path.abspath(datadir) + self.filelist = [] + self.times = [] + self.verbose = kwargs.get('verbose',False) + + def __len__(self): + return len(self.filelist) + + def __getitem__(self,i): + return self.filelist[i] + + def __iter__(self): + self.lastfile = -1 # reset iterator index + return self + + def __next__(self): + if self.filelist is None: + raise StopIteration('file list is empty') + self.lastfile += 1 + if self.lastfile >= self.Ntimes: + raise StopIteration + else: + return self.filelist[self.lastfile] + + def next(self): + # for Python 2 compatibility + return self.__next__() + + def itertimes(self): + return zip(self.times,self.filelist) + + def trimtimes(self,tstart=None,tend=None): + if (tstart is not None) or (tend is not None): + if tstart is None: tstart = 0.0 + if tend is None: tend = 9e9 + selected = [ (t >= tstart) & (t <= tend) for t in self.times ] + self.filelist = [self.filelist[i] for i,b in enumerate(selected) if b ] + self.times = [self.times[i] for i,b in enumerate(selected) if b ] + self.Ntimes = len(self.times) + + # for SOWFATimeSeries: + try: + self.dirlist = [self.dirlist[i] for i,b in enumerate(selected) if b ] + except AttributeError: + pass + + +class TimeSeries(Series): + """Object for holding time series data in a single directory + + Written by Eliot Quon (eliot.quon@nrel.gov) + + Sample usage: + from datatools.series import TimeSeries + #ts = TimeSeries('/path/to/data',prefix='foo',suffix='.bar') + ts = TimeSeries('/path/to/data',prefix='foo',suffix='.bar', + tstart=21200,tend=21800) + for fname in ts: + do_something(fname) + for t,fname in ts.itertimes(): + do_something(t,fname) + """ + + def __init__(self, + datadir='.', + prefix='', suffix='', + dt=1.0, t0=0.0, + dirs=False, + tstart=None, tend=None, + **kwargs): + """Collect data from specified directory, for files with a + given prefix and optional suffix. For series with integer time + step numbers, dt can be specified (with t0 offset) to determine + the time for each snapshot. + """ + super(self.__class__,self).__init__(datadir,**kwargs) + self.dt = dt + self.t0 = t0 + + if dirs: + def check_path(f): return os.path.isdir(f) + else: + def check_path(f): return os.path.isfile(f) + + if self.verbose: + print('Retrieving time series from',self.datadir) + + for f in os.listdir(self.datadir): + if (check_path(os.path.join(self.datadir,f))) \ + and f.startswith(prefix) \ + and f.endswith(suffix): + fpath = os.path.join(self.datadir,f) + self.filelist.append(fpath) + val = f[len(prefix):] + if len(suffix) > 0: + val = val[:-len(suffix)] + try: + self.times.append(t0 + dt*float(val)) + except ValueError: + print('Prefix and/or suffix are improperly specified') + print(' attempting to cast value: '+val) + print(' for file: '+fpath) + break + self.Ntimes = len(self.filelist) + if self.Ntimes == 0: + print('Warning: no matching files were found') + + # sort by output time + iorder = [kv[0] for kv in sorted(enumerate(self.times),key=lambda x:x[1])] + self.filelist = [self.filelist[i] for i in iorder] + self.times = [self.times[i] for i in iorder] + + # select time range + self.trimtimes(tstart,tend) + + +class SOWFATimeSeries(Series): + """Object for holding general time series data stored in multiple + time subdirectories, e.g., as done in OpenFOAM. + + Written by Eliot Quon (eliot.quon@nrel.gov) + + Sample usage: + from datatools.series import SOWFATimeSeries + ts = SOWFATimeSeries('/path/to/data',filename='U') + """ + + def __init__(self,datadir='.',filename=None,**kwargs): + """Collect data from subdirectories, assuming that subdirs + have a name that can be cast as a float + """ + super(self.__class__,self).__init__(datadir,**kwargs) + self.dirlist = [] + self.timenames = [] + self.filename = filename + + if self.verbose: + print('Retrieving SOWFA time series from',self.datadir) + + # process all subdirectories + subdirs = [ os.path.join(self.datadir,d) + for d in os.listdir(self.datadir) + if os.path.isdir(os.path.join(self.datadir,d)) ] + for path in subdirs: + dname = os.path.split(path)[-1] + try: + tval = float(dname) + except ValueError: + continue + self.times.append(tval) + self.dirlist.append(path) + self.Ntimes = len(self.dirlist) + + # sort by output time + iorder = [kv[0] for kv in sorted(enumerate(self.times),key=lambda x:x[1])] + self.dirlist = [self.dirlist[i] for i in iorder] + self.times = [self.times[i] for i in iorder] + + # check that all subdirectories contain the same files + self.timenames = os.listdir(self.dirlist[0]) + for d in self.dirlist: + if not os.listdir(d) == self.timenames: + print('Warning: not all subdirectories contain the same files') + break + if self.verbose: + self.outputs() # print available outputs + + # set up file list + if filename is not None: + self.get(filename) + + # select time range + tstart = kwargs.get('tstart',None) + tend = kwargs.get('tend',None) + self.trimtimes(tstart,tend) + + def get(self,filename): + """Update file list for iteration""" + self.filelist = [] + for path in self.dirlist: + fpath = os.path.join(path,filename) + if os.path.isfile(fpath): + self.filelist.append(fpath) + else: + raise IOError(fpath+' not found') + + def outputs(self,prefix=''): + """Print available outputs for the given data directory""" + selected_output_names = [ name for name in self.timenames if name.startswith(prefix) ] + if self.verbose: + if prefix: + print('Files starting with "{}" in each subdirectory:'.format(prefix)) + else: + print('Files in each subdirectory:') + #print('\t'.join([ ' '+name for name in selected_output_names ])) + print(pretty_list(sorted(selected_output_names))) + return selected_output_names + + def __repr__(self): + return str(self.Ntimes) + ' time subdirectories located in ' + self.datadir + diff --git a/windtools/io/vtk.py b/windtools/io/vtk.py new file mode 100644 index 0000000..eaefc77 --- /dev/null +++ b/windtools/io/vtk.py @@ -0,0 +1,171 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +Functions to deal with simple VTK I/O. +""" +import struct + +def vtk_write_structured_points( f, + datadict, + ds=None,dx=None,dy=None,dz=None, + origin=(0.0,0.0,0.0), + indexorder='ijk', + vtk_header='# vtk DataFile Version 2.0', + vtk_datatype='float', + vtk_description='really cool data' + ): + """Write a VTK dataset with regular topology to file handle 'f' + written by Eliot Quon (eliot.quon@nrel.gov) + + Inputs are written with x increasing fastest, then y, then z. + + Example: Writing out two vector fields in one VTK file. + ``` + from windtools.io.vtk import vtk_write_structured_points + with open('some_data.vtk','wb') as f: + vtk_write_structured_points(f, + {'vel':np.stack((u,v,w))}, + ds=1.0, + indexorder='ijk') + ``` + + Parameters + ---------- + datadict : dict + Dictionary with keys as the data names. Data are either scalar + fields with shape (nx,ny,nz) or vector fields with shape + (3,nx,ny,nz). + ds : float, optional + Default grid spacing; dx,dy,dz may be specified to override + dx, dy, dz : float, optional + Specific grid spacings; if ds is not specified, then all three + must be specified + origin : list-like, optional + Origin of the grid + indexorder: str + Specify the indexing convention (standard: 'ijk', TTUDD: 'jik') + + @author: ewquon + """ + # calculate grid spacings if needed + if ds: + if not dx: dx = ds + if not dy: dy = ds + if not dz: dz = ds + else: + assert( dx > 0 and dy > 0 and dz > 0 ) + + # check data + nx = ny = nz = None + datatype = {} + for name,data in datadict.items(): + dims = data.shape + if len(dims) == 3: + datatype[name] = 'scalar' + elif len(dims) == 4: + assert dims[0] == 3 + datatype[name] = 'vector' + else: + raise ValueError('Unexpected "'+name+'" array shape: '+str(data.shape)) + if nx is None: + nx = dims[-3] + ny = dims[-2] + nz = dims[-1] + else: + assert (nx==dims[-3]) and (ny==dims[-2]) and (nz==dims[-1]) + + # write header + if 'b' in f.mode: + binary = True + import struct + if bytes is str: + # python 2 + def b(s): + return str(s) + else: + # python 3 + def b(s): + return bytes(s,'utf-8') + f.write(b(vtk_header+'\n')) + f.write(b(vtk_description+'\n')) + f.write(b('BINARY\n')) + f.write(b('DATASET STRUCTURED_POINTS\n')) + + # write out mesh descriptors + f.write(b('DIMENSIONS {:d} {:d} {:d}\n'.format(nx,ny,nz))) + f.write(b('ORIGIN {:f} {:f} {:f}\n'.format(origin[0],origin[1],origin[2]))) + f.write(b('SPACING {:f} {:f} {:f}\n'.format(dx,dy,dz))) + + # write out data + f.write(b('POINT_DATA {:d}\n'.format(nx*ny*nz))) + + else: + binary = False + f.write(vtk_header+'\n') + f.write(vtk_description+'\n') + f.write('ASCII\n') + f.write('DATASET STRUCTURED_POINTS\n') + + # write out mesh descriptors + f.write('DIMENSIONS {:d} {:d} {:d}\n'.format(nx,ny,nz)) + f.write('ORIGIN {:f} {:f} {:f}\n'.format(origin[0],origin[1],origin[2])) + f.write('SPACING {:f} {:f} {:f}\n'.format(dx,dy,dz)) + + # write out data + f.write('POINT_DATA {:d}\n'.format(nx*ny*nz)) + + for name,data in datadict.items(): + outputtype = datatype[name] + if outputtype=='vector': + u = data[0,:,:,:] + v = data[1,:,:,:] + w = data[2,:,:,:] + elif outputtype=='scalar': + u = data + else: + raise ValueError('Unexpected data type '+outputtype) + + name = name.replace(' ','_') + + mapping = { 'i': range(nx), 'j': range(ny), 'k': range(nz) } + ijkranges = [ mapping[ijk] for ijk in indexorder ] + + if outputtype=='vector': + if binary: + f.write(b('{:s}S {:s} {:s}\n'.format(outputtype.upper(),name,vtk_datatype))) + for k in ijkranges[2]: + for j in ijkranges[1]: + for i in ijkranges[0]: + f.write(struct.pack('>fff', u[i,j,k], v[i,j,k], w[i,j,k])) # big endian + else: #ascii + f.write('{:s}S {:s} {:s}\n'.format(outputtype.upper(),name,vtk_datatype)) + for k in ijkranges[2]: + for j in ijkranges[1]: + for i in ijkranges[0]: + f.write(' {:f} {:f} {:f}\n'.format(u[i,j,k], v[i,j,k], w[i,j,k])) + elif outputtype=='scalar': + if binary: + f.write(b('{:s}S {:s} {:s}\n'.format(outputtype.upper(),name,vtk_datatype))) + f.write(b('LOOKUP_TABLE default\n')) + for k in ijkranges[2]: + for j in ijkranges[1]: + for i in ijkranges[0]: + #f.write(struct.pack('f',u[j,i,k])) # native endianness + f.write(struct.pack('>f',u[i,j,k])) # big endian + else: + f.write('{:s}S {:s} {:s}\n'.format(outputtype.upper(),name,vtk_datatype)) + f.write('LOOKUP_TABLE default\n') + for k in ijkranges[2]: + for j in ijkranges[1]: + for i in ijkranges[0]: + f.write(' {:f}\n'.format(u[i,j,k])) + diff --git a/windtools/openfast.py b/windtools/openfast.py new file mode 100644 index 0000000..e71595c --- /dev/null +++ b/windtools/openfast.py @@ -0,0 +1,130 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import os +import struct + +channelnames = { + 'GenPwr_[kW]': 'Generator Power [kW]', + 'GenTq_[kN-m]': 'Generator Torque [kN-m]', + 'RotSpeed_[rpm]': 'Rotor Speed [RPM]', + 'BldPitch1_[deg]': 'Blade 1 Pitch [deg]', + 'BldPitch2_[deg]': 'Blade 2 Pitch [deg]', + 'BldPitch3_[deg]': 'Blade 3 Pitch [deg]', + 'YawPos_[deg]': 'Nacelle Yaw Position [deg]', +} + +InflowWind_template = """------- InflowWind v3.01.* INPUT FILE ------------------------------------------------------------------------- +Inflow description here. +--------------------------------------------------------------------------------------------------------------- +False Echo - Echo input data to .ech (flag) +{WindType:>11d} WindType - switch for wind file type (1=steady; 2=uniform; 3=binary TurbSim FF; 4=binary Bladed-style FF; 5=HAWC format; 6=User defined) + 0 PropagationDir - Direction of wind propagation (meteoroligical rotation from aligned with X (positive rotates towards -Y) -- degrees) + 1 NWindVel - Number of points to output the wind velocity (0 to 9) + 0 WindVxiList - List of coordinates in the inertial X direction (m) + 0 WindVyiList - List of coordinates in the inertial Y direction (m) +{RefHt:>11f} WindVziList - List of coordinates in the inertial Z direction (m) +================== Parameters for Steady Wind Conditions [used only for WindType = 1] ========================= +{URef:>11f} HWindSpeed - Horizontal windspeed (m/s) +{RefHt:>11f} RefHt - Reference height for horizontal wind speed (m) + 0 PLexp - Power law exponent (-) +================== Parameters for Uniform wind file [used only for WindType = 2] ============================ +"unused" Filename - Filename of time series data for uniform wind field. (-) +{RefHt:>11f} RefHt - Reference height for horizontal wind speed (m) + 125.88 RefLength - Reference length for linear horizontal and vertical sheer (-) +================== Parameters for Binary TurbSim Full-Field files [used only for WindType = 3] ============== +"unused" Filename - Name of the Full field wind file to use (.bts) +================== Parameters for Binary Bladed-style Full-Field files [used only for WindType = 4] ========= +"unused" FilenameRoot - Rootname of the full-field wind file to use (.wnd, .sum) +False TowerFile - Have tower file (.twr) (flag) +================== Parameters for HAWC-format binary files [Only used with WindType = 5] ===================== +"{hawc_ufile:s}" FileName_u - name of the file containing the u-component fluctuating wind (.bin) +"{hawc_vfile:s}" FileName_v - name of the file containing the v-component fluctuating wind (.bin) +"{hawc_wfile:s}" FileName_w - name of the file containing the w-component fluctuating wind (.bin) +{nx:>11d} nx - number of grids in the x direction (in the 3 files above) (-) +{ny:>11d} ny - number of grids in the y direction (in the 3 files above) (-) +{nz:>11d} nz - number of grids in the z direction (in the 3 files above) (-) +{dx:>11f} dx - distance (in meters) between points in the x direction (m) +{dy:>11f} dy - distance (in meters) between points in the y direction (m) +{dz:>11f} dz - distance (in meters) between points in the z direction (m) +{RefHt:>11f} RefHt - reference height; the height (in meters) of the vertical center of the grid (m) + ------------- Scaling parameters for turbulence --------------------------------------------------------- + 0 ScaleMethod - Turbulence scaling method [0 = none, 1 = direct scaling, 2 = calculate scaling factor based on a desired standard deviation] + 0 SFx - Turbulence scaling factor for the x direction (-) [ScaleMethod=1] + 0 SFy - Turbulence scaling factor for the y direction (-) [ScaleMethod=1] + 0 SFz - Turbulence scaling factor for the z direction (-) [ScaleMethod=1] + 0 SigmaFx - Turbulence standard deviation to calculate scaling from in x direction (m/s) [ScaleMethod=2] + 0 SigmaFy - Turbulence standard deviation to calculate scaling from in y direction (m/s) [ScaleMethod=2] + 0 SigmaFz - Turbulence standard deviation to calculate scaling from in z direction (m/s) [ScaleMethod=2] + ------------- Mean wind profile parameters (added to HAWC-format files) --------------------------------- +{URef:>11f} URef - Mean u-component wind speed at the reference height (m/s) + -1 WindProfile - Wind profile type (0=constant;1=logarithmic,2=power law) + 0 PLExp - Power law exponent (-) (used for PL wind profile type only) + 0.0 Z0 - Surface roughness length (m) (used for LG wind profile type only) +====================== OUTPUT ================================================== +True SumPrint - Print summary data to .IfW.sum (flag) + OutList - The next line(s) contains a list of output parameters. See OutListParameters.xlsx for a listing of available output channels, (-) +"Wind1VelX" X-direction wind velocity at point WindList(1) +"Wind1VelY" Y-direction wind velocity at point WindList(1) +"Wind1VelZ" Z-direction wind velocity at point WindList(1) +END of input file (the word "END" must appear in the first 3 columns of this last OutList line) +---------------------------------------------------------------------------------------""" + +def to_InflowWind(ds, outdir='.', prefix=''): + """Write out Binary HAWC-Style Full-Field Files + + From the InflowWind manual: + ``` + HAWC FF files are 3-dimensional data sets (functions of x, y, and z) of the + 3-component wind inflow velocity, u, v, w. The data are stored in a nx × ny + × nz evenly-spaced grid, V(x,y,z). + + HAWC-style binary files do not contain any header information. All data + necessary to read and scale it must be entered in the InflowWind input file. + Each data file contains the wind speed for a specific wind component, stored + as 4-byte real numbers. + ``` + + These were selected as the preferred input format due to the format + simplicity. This function generates u.bin, v.bin, and w.bin from an xarray + Dataset with dimensions 't', 'y', and 'z' in a turbine frame of reference + (x is streamwise, z is up) and with variables 'u','v', and 'w' corresponding + to the velocity components. + """ + dims = ['t','y','z'] + ds = ds.sortby(dims).transpose(*dims) + Nt = ds.dims['t'] + Ny = ds.dims['y'] + Nz = ds.dims['z'] + fmtstr = '{:d}f'.format(Nz) + for varname in ['u','v','w']: + fpath = os.path.join(outdir, prefix+varname+'.bin') + with open(fpath,'wb') as f: + data = ds[varname].values + # file format (from InflowWind manual): + # + # ix = nx, nx-1, ... 1 + # iy = ny, ny-1, ... 1 + # iz = 1, 2, ... nz + # Vgrid(iz,iy,ix,i) + # end iz + # end iy + # end ix + # + # where looping over ix can be replaced with + # it = 1, 2, ... nt + # because the last (nx) plane in the inflow box corresponds to the + # first inflow timestep. + for i in range(Nt): + for j in range(Ny)[::-1]: + f.write(struct.pack(fmtstr, *data[i,j,:])) + print('Wrote',fpath) + diff --git a/windtools/openfoam.py b/windtools/openfoam.py new file mode 100644 index 0000000..d32899a --- /dev/null +++ b/windtools/openfoam.py @@ -0,0 +1,294 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import re + + +class InputFile(dict): + """Object to parse and store openfoam input file data + + Written by Eliot Quon (eliot.quon@nrel.gov) + + Includes support for parsing: + - single values, with attempted cast to float/bool + - lists + - dictionaries + """ + DEBUG = False + + block_defs = [ + ('{','}',dict), + ('(',')',list), + ('[',']',list), + ] + true_values = [ + 'true', + 'on', + 'yes', + ] + false_values = [ + 'false', + 'off', + 'no', + 'none', + ] + special_keywords = [ + 'uniform', + 'nonuniform', + 'table', + ] + + def __init__(self,fpath,nodef=False): + """Create a dictionary of definitions from an OpenFOAM-style + input file. + + Inputs + ------ + fpath : str + Path to OpenFOAM file + nodef : bool, optional + If the file only contains OpenFOAM data, e.g., a table of + vector values to be included from another OpenFOAM file, + then create a generic 'data' parent object to contain the + file data. + """ + # read full file + with open(fpath) as f: + lines = f.readlines() + if nodef: + lines = ['data ('] + lines + [')'] + # trim single-line comments and remove directives + for i,line in enumerate(lines): + line = line.strip() + if line.startswith('#'): + if self.DEBUG: + print('Ignoring directive:',line) + lines[i] = '' + else: + idx = line.find('//') + if idx >= 0: + lines[i] = line[:idx].strip() + # trim multi-line comments + txt = '\n'.join(lines) + idx0 = txt.find('/*') + while idx0 >= 0: + idx1 = txt.find('*/',idx0+1) + assert (idx1 > idx0), 'Mismatched comment block' + if self.DEBUG: + print('Remove comment block:',txt[idx0:idx1]) + txt = txt[:idx0] + txt[idx1+2:] + idx0 = txt.find('/*') + # consolidate definitions into single lines + txt = txt.replace('\n',' ') + txt = txt.replace('\t',' ') + txt = txt.strip() + # now parse each line + for name,line,containertype in self._split_defs(txt): + if self.DEBUG: + print('\nPARSING',name,'FROM',line,'of TYPE',containertype) + self._parse(name,line,containertype) + self._sanitycheck() + + def _sanitycheck(self): + """Make sure the InputFile was read properly""" + noparent = [key is None for key in self.keys()] + if any(noparent): + print('Definitions improperly read, some values without keys') + print('If you believe this is an error, then re-run with the nodef keyword') + + def _format_item_str(self,val,maxstrlen=60): + printval = str(val) + if isinstance(val,list) and (len(printval) > maxstrlen): + printval = '[list of length {:d}]'.format(len(val)) + return printval + + def __repr__(self): + descstrs = [ + '{:s} : {:s}'.format(key, self._format_item_str(val)) + for key,val in self.items() + ] + return '\n'.join(descstrs) + + def _split_defs(self,txt): + """Splits blocks of text into lines in the following forms: + key value; + key (values...) + key {values...} + (values...) + ((values...) (values...)) + where lists and dicts may be nested. The outlier case is the + (nested) list which takes on the key of its parent. + """ + names, lines, container = [], [], [] + while len(txt) > 0: + if self.DEBUG: + print('current text:',txt) + + if (txt[0] == '('): + # special treatment for lists, or lists within a list + name = None + else: + # - find first word (name) + idx = txt.find(' ') + name = txt[:idx] + if self.DEBUG: print('name=',name) + txt = txt[idx+1:].strip() + + # - find next word (either a value/block) + idx = txt.find(' ') + if idx < 0: + # EOF + string = txt + txt = '' # to exit loop + if self.DEBUG: print('EOF',string) + else: + string = txt[:idx].strip() + if string in self.special_keywords: + # append special keyword to name and read the next word + name += '_'+string + txt = txt[idx+1:].strip() + idx = txt.find(' ') + assert (idx > 0), 'problem parsing '+string+' field' + string = txt[:idx].strip() + + if string.endswith(';'): + # found single definition + if self.DEBUG: print('value=',string[:-1]) + names.append(name) + lines.append(string[:-1]) # strip ; + container.append(None) + else: + # found block + if self.DEBUG: print('current string:',string) + blockstart = string[0] + blockend = None + blocktype = None + for block in self.block_defs: + if blockstart == block[0]: + blockend = block[1] + blocktype = block[2] + break + assert (blockend is not None), 'Unknown input block '+blockstart + # find end of block + idx = txt.find(blockend) + 1 + assert (idx > 0), 'Mismatched input block' + # consolidate spaces + blockdef = re.sub(' +',' ',txt[:idx].strip()) + Nopen = blockdef.count(blockstart) + Nclose = blockdef.count(blockend) + while Nopen != Nclose: + if self.DEBUG: + print(' incomplete:',blockdef) + idx = txt.find(blockend, idx) + 1 + blockdef = txt[:idx].strip() + Nopen = blockdef.count(blockstart) + Nclose = blockdef.count(blockend) + # select block + if self.DEBUG: print('complete block=',blockdef) + names.append(name) + lines.append(blockdef) + container.append(blocktype) + if self.DEBUG: print('container type=',container[-1]) + # trim text block + txt = txt[idx+1:].strip() + + return zip(names, lines, container) + + def _parse(self,name,defn,containertype,parent=None): + """Parse values split up by _split_defs() + + Casts to float and bool (the latter by checking against a list + of known true/false values, since bool(some_str) will return + True if the string has a nonzero length) will be attempted. + + If the value is a container (i.e., list or dict), then + _split_defs() and _parse() will be called recursively. + """ + if self.DEBUG: + print('----------- parsing block -----------') + if parent is not None: + print('name:',name,'parent:',str(parent)) + if containertype is not None: + print('container type:',containertype) + defn = defn.strip() + if containertype is None: + # set single value in parent + defn = self._try_cast(defn) + # SET VALUE HERE + if self.DEBUG: + print(name,'-->',defn) + if parent is None: + self.__setitem__(name, defn) + elif isinstance(parent, dict): + parent[name] = defn + else: + assert isinstance(parent, list) + parent.append(defn) + else: + # we have a subblock, create new container + if parent is None: + # parent is the InputFile object + if self.DEBUG: + print('CREATING',containertype,'named',name) + self.__setitem__(name, containertype()) + newparent = self.__getitem__(name) + elif isinstance(parent, dict): + # parent is a dictionary + if self.DEBUG: + print('ADDING dictionary entry,',name) + parent[name] = containertype() + newparent = parent[name] + else: + assert isinstance(parent, list) + # parent is a list + if self.DEBUG: + print('ADDING list item, name=',name) + if name is not None: + # if we have nested nists with mixed types we could + # end up here... + parent.append(self._try_cast(name)) + newparent = containertype() + parent.append(newparent) + newdefn = defn[1:-1].strip() + if (containertype is list) \ + and ('(' not in newdefn) and (')' not in newdefn): + # special treatment for lists + for val in newdefn.split(): + # recursively call parse wihout a name (None for + # list) and without a container type to indicate + # that a new value should be set + self._parse(None,val,None,parent=newparent) + else: + for newname,newdef,newcontainertype in self._split_defs(newdefn): + self._parse(newname,newdef,newcontainertype,parent=newparent) + + def _try_cast(self,s): + assert(s.find(' ') < 0) + try: + # attempt float cast + s = float(s) + except ValueError: + # THIS IS A TRAP + #try: + # # attempt boolean cast + # s = bool(s) + #except ValueError: + # # default to string + # pass + if s.lower() in self.true_values: + s = True + elif s.lower() in self.false_values: + s = False + else: + # default to string + s = s.strip('"') + s = s.strip('\'') + return s diff --git a/windtools/plotting.py b/windtools/plotting.py new file mode 100644 index 0000000..da9e84d --- /dev/null +++ b/windtools/plotting.py @@ -0,0 +1,2154 @@ +# Copyright 2019 NREL + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +""" +Library of standardized plotting functions for basic plot formats + +Written by Dries Allaerts (dries.allaerts@nrel.gov) +""" + +import numpy as np +import matplotlib as mpl +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +import pandas as pd +import xarray as xr +from scipy.interpolate import interp1d +from scipy.signal import welch + +# Standard field labels +# - default: e.g., "Km/s" +# - all superscript: e.g., "K m s^{-1}" +fieldlabels_default_units = { + 'wspd': r'Wind speed [m/s]', + 'wdir': r'Wind direction [$^\circ$]', + 'u': r'u [m/s]', 'Ux': r'$u$ [m/s]', + 'v': r'v [m/s]', 'Uy': r'$v$ [m/s]', + 'w': r'Vertical wind speed [m/s]', 'Uz': r'Vertical wind speed [m/s]', + 'theta': r'$\theta$ [K]', 'T': r'$\theta$ [K]', + 'thetav': r'$\theta_v$ [K]', + 'uu': r'$\langle u^\prime u^\prime \rangle \;[\mathrm{m^2/s^2}]$', 'UUxx': r'$\langle u^\prime u^\prime \rangle \;[\mathrm{m^2/s^2}]$', + 'vv': r'$\langle v^\prime v^\prime \rangle \;[\mathrm{m^2/s^2}]$', 'UUyy': r'$\langle v^\prime v^\prime \rangle \;[\mathrm{m^2/s^2}]$', + 'ww': r'$\langle w^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', 'UUzz': r'$\langle w^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', + 'uv': r'$\langle u^\prime v^\prime \rangle \;[\mathrm{m^2/s^2}]$', 'UUxy': r'$\langle u^\prime v^\prime \rangle \;[\mathrm{m^2/s^2}]$', + 'uw': r'$\langle u^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', 'UUxz': r'$\langle u^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', + 'vw': r'$\langle v^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', 'UUyz': r'$\langle v^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', + 'tw': r'$\langle w^\prime \theta^\prime \rangle \;[\mathrm{Km/s}]$', 'TUz': r'$\langle w^\prime \theta^\prime \rangle \;[\mathrm{Km/s}]$', + 'TI': r'TI $[-]$', + 'TKE': r'TKE $[\mathrm{m^2/s^2}]$', +} +fieldlabels_superscript_units = { + 'wspd': r'Wind speed [m s$^{-1}$]', + 'wdir': r'Wind direction [$^\circ$]', + 'u': r'u [m s$^{-1}$]', 'Ux': r'$u$ [m s$^{-1}$]', + 'v': r'v [m s$^{-1}$]', 'Uy': r'$v$ [m s$^{-1}$]', + 'w': r'Vertical wind speed [m s$^{-1}$]', 'Uz': r'Vertical wind speed [m s$^{-1}$]', + 'theta': r'$\theta$ [K]', 'T': r'$\theta$ [K]', + 'thetav': r'$\theta_v$ [K]', + 'uu': r'$\langle u^\prime u^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', 'UUxx': r'$\langle u^\prime u^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', + 'vv': r'$\langle v^\prime v^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', 'UUyy': r'$\langle v^\prime v^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', + 'ww': r'$\langle w^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', 'UUzz': r'$\langle w^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', + 'uv': r'$\langle u^\prime v^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', 'UUxy': r'$\langle u^\prime v^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', + 'uw': r'$\langle u^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', 'UUxz': r'$\langle u^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', + 'vw': r'$\langle v^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', 'UUyz': r'$\langle v^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', + 'tw': r'$\langle w^\prime \theta^\prime \rangle \;[\mathrm{K m s^{-1}}]$', 'TUz': r'$\langle w^\prime \theta^\prime \rangle \;[\mathrm{K m s^{-1}}]$', + 'TI': r'TI $[-]$', + 'TKE': r'TKE $[\mathrm{m^2 s^{-2}}]$', +} + +# Standard field labels for frequency spectra +spectrumlabels_default_units = { + 'u': r'$E_{uu}\;[\mathrm{m^2/s}]$', + 'v': r'$E_{vv}\;[\mathrm{m^2/s}]$', + 'w': r'$E_{ww}\;[\mathrm{m^2/s}]$', + 'theta': r'$E_{\theta\theta}\;[\mathrm{K^2 s}]$', + 'thetav': r'$E_{\theta\theta}\;[\mathrm{K^2 s}]$', + 'wspd': r'$E_{UU}\;[\mathrm{m^2/s}]$', +} +spectrumlabels_superscript_units = { + 'u': r'$E_{uu}\;[\mathrm{m^2\;s^{-1}}]$', + 'v': r'$E_{vv}\;[\mathrm{m^2\;s^{-1}}]$', + 'w': r'$E_{ww}\;[\mathrm{m^2\;s^{-1}}]$', + 'theta': r'$E_{\theta\theta}\;[\mathrm{K^2\;s}]$', + 'thetav': r'$E_{\theta\theta}\;[\mathrm{K^2\;s}]$', + 'wspd': r'$E_{UU}\;[\mathrm{m^2\;s^{-1}}]$', +} + +# Default settings +default_colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] +standard_fieldlabels = fieldlabels_default_units +standard_spectrumlabels = spectrumlabels_default_units + +# Supported dimensions and associated names +dimension_names = { + 'time': ['datetime','time','Time','t'], + 'height': ['height','heights','z','zagl'], + 'frequency': ['frequency','f',] +} + +# Show debug information +debug = False + +def plot_timeheight(datasets, + fields=None, + fig=None,ax=None, + colorschemes={}, + fieldlimits=None, + heightlimits=None, + timelimits=None, + fieldlabels={}, + labelsubplots=False, + showcolorbars=True, + fieldorder='C', + ncols=1, + subfigsize=(12,4), + plot_local_time=False, + local_time_offset=0, + datasetkwargs={}, + **kwargs + ): + """ + Plot time-height contours for different datasets and fields + + Usage + ===== + datasets : pandas.DataFrame or dict + Dataset(s). If more than one set, datasets should + be a dictionary with entries : dataset + fields : str, list, 'all' (or None) + Fieldname(s) corresponding to particular column(s) of + the datasets. fields can be None if input are MultiIndex Series. + 'all' means all fields will be plotted (in this case all + datasets should have the same fields) + fig : figure handle + Custom figure handle. Should be specified together with ax + ax : axes handle, or list or numpy ndarray with axes handles + Customand axes handle(s). + Size of ax should equal ndatasets*nfields + colorschemes : str or dict + Name of colorschemes. If only one field is plotted, colorschemes + can be a string. Otherwise, it should be a dictionary with + entries : name_of_colorschemes + Missing colorschemess are set to 'viridis' + fieldlimits : list or tuple, or dict + Value range for the various fields. If only one field is + plotted, fieldlimits can be a list or tuple. Otherwise, it + should be a dictionary with entries : fieldlimit. + Missing fieldlimits are set automatically + heightlimits : list or tuple + Height axis limits + timelimits : list or tuple + Time axis limits + fieldlabels : str or dict + Custom field labels. If only one field is plotted, fieldlabels + can be a string. Otherwise it should be a dictionary with + entries : fieldlabel + labelsubplots : bool, list or tuple + Label subplots as (a), (b), (c), ... If a list or tuple is given + their values should be the horizontal and vertical position + relative to each subaxis. + showcolorbars : bool + Show colorbar per subplot + fieldorder : 'C' or 'F' + Index ordering for assigning fields and datasets to axes grid + (row by row). Fields is considered the first axis, so 'C' means + fields change slowest, 'F' means fields change fastest. + ncols : int + Number of columns in axes grid, must be a true divisor of total + number of axes. + subfigsize : list or tuple + Standard size of subfigures + plot_local_time : bool or str + Plot dual x axes with both UTC time and local time. If a str is + provided, then plot_local_time is assumed to be True and the str + is used as the datetime format. + local_time_offset : float + Local time offset from UTC + datasetkwargs : dict + Dataset-specific options that are passed on to the actual + plotting function. These options overwrite general options + specified through **kwargs. The argument should be a dictionary + with entries : {**kwargs} + **kwargs : other keyword arguments + Options that are passed on to the actual plotting function. + Note that these options should be the same for all datasets and + fields and can not be used to set dataset or field specific + limits, colorschemess, norms, etc. + Example uses include setting shading, rasterized, etc. + """ + + args = PlottingInput( + datasets=datasets, + fields=fields, + fieldlimits=fieldlimits, + fieldlabels=fieldlabels, + colorschemes=colorschemes, + fieldorder=fieldorder + ) + args.set_missing_fieldlimits() + + nfields = len(args.fields) + ndatasets = len(args.datasets) + ntotal = nfields * ndatasets + + # Concatenate custom and standard field labels + # (custom field labels overwrite standard fields labels if existent) + args.fieldlabels = {**standard_fieldlabels, **args.fieldlabels} + + fig, ax, nrows, ncols = _create_subplots_if_needed( + ntotal, + ncols, + sharex=True, + sharey=True, + subfigsize=subfigsize, + hspace=0.2, + fig=fig, + ax=ax + ) + + # Create flattened view of axes + axv = np.asarray(ax).reshape(-1) + + # Initialise list of colorbars + cbars = [] + + # Loop over datasets, fields and times + for i, dfname in enumerate(args.datasets): + df = args.datasets[dfname] + + heightvalues = _get_dim_values(df,'height') + timevalues = _get_dim_values(df,'time') + assert(heightvalues is not None), 'timeheight plot needs a height axis' + assert(timevalues is not None), 'timeheight plot needs a time axis' + + if isinstance(timevalues, pd.DatetimeIndex): + # If plot local time, shift timevalues + if plot_local_time is not False: + timevalues = timevalues + pd.to_timedelta(local_time_offset,'h') + + # Convert to days since 0001-01-01 00:00 UTC, plus one + numerical_timevalues = mdates.date2num(timevalues.values) + else: + if isinstance(timevalues, pd.TimedeltaIndex): + timevalues = timevalues.total_seconds() + + # Timevalues is already a numerical array + numerical_timevalues = timevalues + + # Create time-height mesh grid + tst = _get_staggered_grid(numerical_timevalues) + zst = _get_staggered_grid(heightvalues) + Ts,Zs = np.meshgrid(tst,zst,indexing='xy') + + # Create list with available fields only + available_fields = _get_available_fieldnames(df,args.fields) + + # Pivot all fields in a dataset at once + df_pivot = _get_pivot_table(df,'height',available_fields) + + for j, field in enumerate(args.fields): + # If available_fields is [None,], fieldname is unimportant + if available_fields == [None]: + pass + # Else, check if field is available + elif not field in available_fields: + print('Warning: field "'+field+'" not available in dataset '+dfname) + continue + + # Store plotting options in dictionary + plotting_properties = { + 'vmin': args.fieldlimits[field][0], + 'vmax': args.fieldlimits[field][1], + 'cmap': args.cmap[field] + } + + # Index of axis corresponding to dataset i and field j + if args.fieldorder=='C': + axi = i*nfields + j + else: + axi = j*ndatasets + i + + # Extract data from dataframe + fieldvalues = _get_pivoted_field(df_pivot,field) + + # Gather label, color, general options and dataset-specific options + # (highest priority to dataset-specific options, then general options) + try: + plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} + except KeyError: + plotting_properties = {**plotting_properties,**kwargs} + + # Plot data + im = axv[axi].pcolormesh(Ts,Zs,fieldvalues.T,**plotting_properties) + + # Colorbar mark up + if showcolorbars: + cbar = fig.colorbar(im,ax=axv[axi],shrink=1.0) + # Set field label if known + try: + cbar.set_label(args.fieldlabels[field]) + except KeyError: + pass + # Save colorbar + cbars.append(cbar) + + # Set title if more than one dataset + if ndatasets>1: + axv[axi].set_title(dfname,fontsize=16) + + + # Format time axis + if isinstance(timevalues, (pd.DatetimeIndex, pd.TimedeltaIndex)): + ax2 = _format_time_axis(fig,axv[(nrows-1)*ncols:],plot_local_time,local_time_offset,timelimits) + else: + ax2 = None + # Set time limits if specified + if not timelimits is None: + axv[-1].set_xlim(timelimits) + # Set time label + for axi in axv[(nrows-1)*ncols:]: + axi.set_xlabel('time [s]') + + if not heightlimits is None: + axv[-1].set_ylim(heightlimits) + + # Add y labels + for r in range(nrows): + axv[r*ncols].set_ylabel(r'Height [m]') + + # Align time, height and color labels + _align_labels(fig,axv,nrows,ncols) + if showcolorbars: + _align_labels(fig,[cb.ax for cb in cbars],nrows,ncols) + + # Number sub figures as a, b, c, ... + if labelsubplots is not False: + try: + hoffset, voffset = labelsubplots + except (TypeError, ValueError): + hoffset, voffset = -0.14, 1.0 + for i,axi in enumerate(axv): + axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) + + # Return cbar instead of array if ntotal==1 + if len(cbars)==1: + cbars=cbars[0] + + if (plot_local_time is not False) and ax2 is not None: + return fig, ax, ax2, cbars + else: + return fig, ax, cbars + + +def plot_timehistory_at_height(datasets, + fields=None, + heights=None, + extrapolate=True, + fig=None,ax=None, + fieldlimits=None, + timelimits=None, + fieldlabels={}, + cmap=None, + stack_by_datasets=None, + labelsubplots=False, + showlegend=None, + ncols=1, + subfigsize=(12,3), + plot_local_time=False, + local_time_offset=0, + datasetkwargs={}, + **kwargs + ): + """ + Plot time history at specified height(s) for various dataset(s) + and/or field(s). + + By default, data for multiple datasets or multiple heights are + stacked in a single subplot. When multiple datasets and multiple + heights are specified together, heights are stacked in a subplot + per field and per dataset. + + Usage + ===== + datasets : pandas.DataFrame or dict + Dataset(s). If more than one set, datasets should + be a dictionary with entries : dataset + fields : str, list, 'all' (or None) + Fieldname(s) corresponding to particular column(s) of + the datasets. fields can be None if input are Series. + 'all' means all fields will be plotted (in this case all + datasets should have the same fields) + heights : float, list, 'all' (or None) + Height(s) for which time history is plotted. heights can be + None if all datasets combined have no more than one height + value. 'all' means the time history for all heights in the + datasets will be plotted (in this case all datasets should + have the same heights) + extrapolate : bool + If false, then output height(s) outside the data range will + not be plotted; default is true for backwards compatibility + fig : figure handle + Custom figure handle. Should be specified together with ax + ax : axes handle, or list or numpy ndarray with axes handles + Customand axes handle(s). + Size of ax should equal nfields * (ndatasets or nheights) + fieldlimits : list or tuple, or dict + Value range for the various fields. If only one field is + plotted, fieldlimits can be a list or tuple. Otherwise, it + should be a dictionary with entries : fieldlimit. + Missing fieldlimits are set automatically + timelimits : list or tuple + Time axis limits + fieldlabels : str or dict + Custom field labels. If only one field is plotted, fieldlabels + can be a string. Otherwise it should be a dictionary with + entries : fieldlabel + cmap : str + Colormap used when stacking heights + stack_by_datasets : bool (or None) + Flag to specify what is plotted ("stacked") together per subfigure. + If True, stack datasets together, otherwise stack by heights. If + None, stack_by_datasets will be set based on the number of heights + and datasets. + labelsubplots : bool, list or tuple + Label subplots as (a), (b), (c), ... If a list or tuple is given + their values should be the horizontal and vertical position + relative to each subaxis. + showlegend : bool (or None) + Label different plots and show legend. If None, showlegend is set + to True if legend will have more than one entry, otherwise it is + set to False. + ncols : int + Number of columns in axes grid, must be a true divisor of total + number of axes. + subfigsize : list or tuple + Standard size of subfigures + plot_local_time : bool or str + Plot dual x axes with both UTC time and local time. If a str is + provided, then plot_local_time is assumed to be True and the str + is used as the datetime format. + local_time_offset : float + Local time offset from UTC + datasetkwargs : dict + Dataset-specific options that are passed on to the actual + plotting function. These options overwrite general options + specified through **kwargs. The argument should be a dictionary + with entries : {**kwargs} + **kwargs : other keyword arguments + Options that are passed on to the actual plotting function. + Note that these options should be the same for all datasets, + fields and heights, and they can not be used to set dataset, + field or height specific colors, limits, etc. + Example uses include setting linestyle/width, marker, etc. + """ + # Avoid FutureWarning concerning the use of an implicitly registered + # datetime converter for a matplotlib plotting method. The converter + # was registered by pandas on import. Future versions of pandas will + # require explicit registration of matplotlib converters, as done here. + from pandas.plotting import register_matplotlib_converters + register_matplotlib_converters() + + args = PlottingInput( + datasets=datasets, + fields=fields, + heights=heights, + fieldlimits=fieldlimits, + fieldlabels=fieldlabels, + ) + + nfields = len(args.fields) + nheights = len(args.heights) + ndatasets = len(args.datasets) + + # Concatenate custom and standard field labels + # (custom field labels overwrite standard fields labels if existent) + args.fieldlabels = {**standard_fieldlabels, **args.fieldlabels} + + # Set up subplot grid + if stack_by_datasets is None: + if nheights>1: + stack_by_datasets = False + else: + stack_by_datasets = True + + if stack_by_datasets: + ntotal = nfields*nheights + else: + ntotal = nfields*ndatasets + + fig, ax, nrows, ncols = _create_subplots_if_needed( + ntotal, + ncols, + sharex=True, + subfigsize=subfigsize, + hspace=0.2, + fig=fig, + ax=ax + ) + + # Create flattened view of axes + axv = np.asarray(ax).reshape(-1) + + # Set showlegend if not specified + if showlegend is None: + if (stack_by_datasets and ndatasets>1) or (not stack_by_datasets and nheights>1): + showlegend = True + else: + showlegend = False + + # Loop over datasets and fields + for i,dfname in enumerate(args.datasets): + df = args.datasets[dfname] + timevalues = _get_dim_values(df,'time',default_idx=True) + assert(timevalues is not None), 'timehistory plot needs a time axis' + heightvalues = _get_dim_values(df,'height') + + if isinstance(timevalues, pd.TimedeltaIndex): + timevalues = timevalues.total_seconds() + + # If plot local time, shift timevalues + if (plot_local_time is not False) and \ + isinstance(timevalues, (pd.DatetimeIndex, pd.TimedeltaIndex)): + timevalues = timevalues + pd.to_timedelta(local_time_offset,'h') + + # Create list with available fields only + available_fields = _get_available_fieldnames(df,args.fields) + + # If any of the requested heights is not available, + # pivot the dataframe to allow interpolation. + # Pivot all fields in a dataset at once to reduce computation time + if (not heightvalues is None) and (not all([h in heightvalues for h in args.heights])): + df_pivot = _get_pivot_table(df,'height',available_fields) + pivoted = True + fill_value = 'extrapolate' if extrapolate else np.nan + if debug: print('Pivoting '+dfname) + else: + pivoted = False + + for j, field in enumerate(args.fields): + # If available_fields is [None,], fieldname is unimportant + if available_fields == [None]: + pass + # Else, check if field is available + elif not field in available_fields: + print('Warning: field "'+field+'" not available in dataset '+dfname) + continue + + for k, height in enumerate(args.heights): + # Check if height is outside of data range + if (heightvalues is not None) and \ + ((height > np.max(heightvalues)) or (height < np.min(heightvalues))): + if extrapolate: + if debug: + print('Extrapolating field "'+field+'" at z='+str(height)+' in dataset '+dfname) + else: + print('Warning: field "'+field+'" not available at z='+str(height)+' in dataset '+dfname) + continue + + # Store plotting options in dictionary + # Set default linestyle to '-' and no markers + plotting_properties = { + 'linestyle':'-', + 'marker':None, + } + + # Axis order, label and title depend on value of stack_by_datasets + if stack_by_datasets: + # Index of axis corresponding to field j and height k + axi = k*nfields + j + + # Use datasetname as label + if showlegend: + plotting_properties['label'] = dfname + + # Set title if multiple heights are compared + if nheights>1: + axv[axi].set_title('z = {:.1f} m'.format(height),fontsize=16) + + # Set colors + plotting_properties['color'] = default_colors[i % len(default_colors)] + else: + # Index of axis corresponding to field j and dataset i + axi = i*nfields + j + + # Use height as label + if showlegend: + plotting_properties['label'] = 'z = {:.1f} m'.format(height) + + # Set title if multiple datasets are compared + if ndatasets>1: + axv[axi].set_title(dfname,fontsize=16) + + # Set colors + if cmap is not None: + cmap = mpl.cm.get_cmap(cmap) + plotting_properties['color'] = cmap(k/(nheights-1)) + else: + plotting_properties['color'] = default_colors[k % len(default_colors)] + + # Extract data from dataframe + if pivoted: + signal = interp1d(heightvalues,_get_pivoted_field(df_pivot,field).values,axis=-1,fill_value=fill_value)(height) + else: + slice_z = _get_slice(df,height,'height') + signal = _get_field(slice_z,field).values + + # Gather label, color, general options and dataset-specific options + # (highest priority to dataset-specific options, then general options) + try: + plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} + except KeyError: + plotting_properties = {**plotting_properties,**kwargs} + + # Plot data + axv[axi].plot(timevalues,signal,**plotting_properties) + + # Set field label if known + try: + axv[axi].set_ylabel(args.fieldlabels[field]) + except KeyError: + pass + # Set field limits if specified + try: + axv[axi].set_ylim(args.fieldlimits[field]) + except KeyError: + pass + + # Set axis grid + for axi in axv: + axi.xaxis.grid(True,which='both') + axi.yaxis.grid(True) + + # Format time axis + if isinstance(timevalues, (pd.DatetimeIndex, pd.TimedeltaIndex)): + ax2 = _format_time_axis(fig,axv[(nrows-1)*ncols:],plot_local_time,local_time_offset,timelimits) + else: + ax2 = None + # Set time limits if specified + if not timelimits is None: + axv[-1].set_xlim(timelimits) + # Set time label + for axi in axv[(nrows-1)*ncols:]: + axi.set_xlabel('time [s]') + + # Number sub figures as a, b, c, ... + if labelsubplots is not False: + try: + hoffset, voffset = labelsubplots + except (TypeError, ValueError): + hoffset, voffset = -0.14, 1.0 + for i,axi in enumerate(axv): + axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) + + # Add legend + if showlegend: + leg = _format_legend(axv,index=ncols-1) + + # Align labels + _align_labels(fig,axv,nrows,ncols) + + if (plot_local_time is not False) and ax2 is not None: + return fig, ax, ax2 + else: + return fig, ax + + +def plot_profile(datasets, + fields=None, + times=None, + timerange=None, + fig=None,ax=None, + fieldlimits=None, + heightlimits=None, + fieldlabels={}, + cmap=None, + stack_by_datasets=None, + labelsubplots=False, + showlegend=None, + fieldorder='C', + ncols=None, + subfigsize=(4,5), + plot_local_time=False, + local_time_offset=0, + datasetkwargs={}, + **kwargs + ): + """ + Plot vertical profile at specified time(s) for various dataset(s) + and/or field(s). + + By default, data for multiple datasets or multiple times are + stacked in a single subplot. When multiple datasets and multiple + times are specified together, times are stacked in a subplot + per field and per dataset. + + Usage + ===== + datasets : pandas.DataFrame or dict + Dataset(s). If more than one set, datasets should + be a dictionary with entries : dataset + fields : str, list, 'all' (or None) + Fieldname(s) corresponding to particular column(s) of + the datasets. fields can be None if input are Series. + 'all' means all fields will be plotted (in this case all + datasets should have the same fields) + times : str, int, float, list (or None) + Time(s) for which vertical profiles are plotted, specified as + either datetime strings or numerical values (seconds, e.g., + simulation time). times can be None if all datasets combined + have no more than one time value, or if timerange is specified. + timerange : tuple or list + Start and end times (inclusive) between which all times are + plotted. If cmap is None, then it will automatically be set to + viridis by default. This overrides times when specified. + fig : figure handle + Custom figure handle. Should be specified together with ax + ax : axes handle, or list or numpy ndarray with axes handles + Customand axes handle(s). + Size of ax should equal nfields * (ndatasets or ntimes) + fieldlimits : list or tuple, or dict + Value range for the various fields. If only one field is + plotted, fieldlimits can be a list or tuple. Otherwise, it + should be a dictionary with entries : fieldlimit. + Missing fieldlimits are set automatically + heightlimits : list or tuple + Height axis limits + fieldlabels : str or dict + Custom field labels. If only one field is plotted, fieldlabels + can be a string. Otherwise it should be a dictionary with + entries : fieldlabel + cmap : str + Colormap used when stacking times + stack_by_datasets : bool (or None) + Flag to specify what is plotted ("stacked") together per subfigure. + If True, stack datasets together, otherwise stack by times. If + None, stack_by_datasets will be set based on the number of times + and datasets. + labelsubplots : bool, list or tuple + Label subplots as (a), (b), (c), ... If a list or tuple is given + their values should be the horizontal and vertical position + relative to each subaxis. + showlegend : bool (or None) + Label different plots and show legend. If None, showlegend is set + to True if legend will have more than one entry, otherwise it is + set to False. + fieldorder : 'C' or 'F' + Index ordering for assigning fields and datasets/times (depending + on stack_by_datasets) to axes grid (row by row). Fields is considered the + first axis, so 'C' means fields change slowest, 'F' means fields + change fastest. + ncols : int + Number of columns in axes grid, must be a true divisor of total + number of axes. + subfigsize : list or tuple + Standard size of subfigures + plot_local_time : bool or str + Plot dual x axes with both UTC time and local time. If a str is + provided, then plot_local_time is assumed to be True and the str + is used as the datetime format. + local_time_offset : float + Local time offset from UTC + datasetkwargs : dict + Dataset-specific options that are passed on to the actual + plotting function. These options overwrite general options + specified through **kwargs. The argument should be a dictionary + with entries : {**kwargs} + **kwargs : other keyword arguments + Options that are passed on to the actual plotting function. + Note that these options should be the same for all datasets, + fields and times, and they can not be used to set dataset, + field or time specific colors, limits, etc. + Example uses include setting linestyle/width, marker, etc. + """ + + args = PlottingInput( + datasets=datasets, + fields=fields, + times=times, + timerange=timerange, + fieldlimits=fieldlimits, + fieldlabels=fieldlabels, + fieldorder=fieldorder, + ) + + nfields = len(args.fields) + ntimes = len(args.times) + ndatasets = len(args.datasets) + + # Concatenate custom and standard field labels + # (custom field labels overwrite standard fields labels if existent) + args.fieldlabels = {**standard_fieldlabels, **args.fieldlabels} + + # Set up subplot grid + if stack_by_datasets is None: + if ntimes>1: + stack_by_datasets = False + else: + stack_by_datasets = True + + if stack_by_datasets: + ntotal = nfields * ntimes + else: + ntotal = nfields * ndatasets + + fig, ax, nrows, ncols = _create_subplots_if_needed( + ntotal, + ncols, + default_ncols=int(ntotal/nfields), + fieldorder=args.fieldorder, + avoid_single_column=True, + sharey=True, + subfigsize=subfigsize, + hspace=0.4, + fig=fig, + ax=ax, + ) + + # Create flattened view of axes + axv = np.asarray(ax).reshape(-1) + + # Set showlegend if not specified + if showlegend is None: + if (stack_by_datasets and ndatasets>1) or (not stack_by_datasets and ntimes>1): + showlegend = True + else: + showlegend = False + + # Set default sequential colormap if timerange was specified + if (timerange is not None) and (cmap is None): + cmap = 'viridis' + + # Loop over datasets, fields and times + for i, dfname in enumerate(args.datasets): + df = args.datasets[dfname] + heightvalues = _get_dim_values(df,'height',default_idx=True) + assert(heightvalues is not None), 'profile plot needs a height axis' + timevalues = _get_dim_values(df,'time') + + # If plot local time, shift timevalues + timedelta_to_local = None + if plot_local_time is not False: + timedelta_to_local = pd.to_timedelta(local_time_offset,'h') + timevalues = timevalues + timedelta_to_local + + # Create list with available fields only + available_fields = _get_available_fieldnames(df,args.fields) + + # Pivot all fields in a dataset at once + if timevalues is not None: + df_pivot = _get_pivot_table(df,'height',available_fields) + + for j, field in enumerate(args.fields): + # If available_fields is [None,], fieldname is unimportant + if available_fields == [None]: + pass + # Else, check if field is available + elif not field in available_fields: + print('Warning: field "'+field+'" not available in dataset '+dfname) + continue + + for k, time in enumerate(args.times): + plotting_properties = {} + + # Axis order, label and title depend on value of stack_by_datasets + if stack_by_datasets: + # Index of axis corresponding to field j and time k + if args.fieldorder == 'C': + axi = j*ntimes + k + else: + axi = k*nfields + j + + # Use datasetname as label + if showlegend: + plotting_properties['label'] = dfname + + # Set title if multiple times are compared + if ntimes>1: + if isinstance(time, (int,float,np.number)): + tstr = '{:g} s'.format(time) + else: + if plot_local_time is False: + tstr = pd.to_datetime(time).strftime('%Y-%m-%d %H%M UTC') + elif plot_local_time is True: + tstr = pd.to_datetime(time).strftime('%Y-%m-%d %H:%M') + else: + assert isinstance(plot_local_time,str), 'Unexpected plot_local_time format' + tstr = pd.to_datetime(time).strftime(plot_local_time) + axv[axi].set_title(tstr, fontsize=16) + + # Set color + plotting_properties['color'] = default_colors[i % len(default_colors)] + else: + # Index of axis corresponding to field j and dataset i + if args.fieldorder == 'C': + axi = j*ndatasets + i + else: + axi = i*nfields + j + + # Use time as label + if showlegend: + if isinstance(time, (int,float,np.number)): + plotting_properties['label'] = '{:g} s'.format(time) + else: + if plot_local_time is False: + plotting_properties['label'] = pd.to_datetime(time).strftime('%Y-%m-%d %H%M UTC') + elif plot_local_time is True: + plotting_properties['label'] = pd.to_datetime(time).strftime('%Y-%m-%d %H:%M') + else: + assert isinstance(plot_local_time,str), 'Unexpected plot_local_time format' + plotting_properties['label'] = pd.to_datetime(time).strftime(plot_local_time) + + # Set title if multiple datasets are compared + if ndatasets>1: + axv[axi].set_title(dfname,fontsize=16) + + # Set colors + if cmap is not None: + cmap = mpl.cm.get_cmap(cmap) + plotting_properties['color'] = cmap(k/(ntimes-1)) + else: + plotting_properties['color'] = default_colors[k % len(default_colors)] + + # Extract data from dataframe + if timevalues is None: + # Dataset will not be pivoted + fieldvalues = _get_field(df,field).values + else: + if plot_local_time is not False: + # specified times are in local time, convert back to UTC + slice_t = _get_slice(df_pivot,time-timedelta_to_local,'time') + else: + slice_t = _get_slice(df_pivot,time,'time') + fieldvalues = _get_pivoted_field(slice_t,field).values.squeeze() + + # Gather label, color, general options and dataset-specific options + # (highest priority to dataset-specific options, then general options) + try: + plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} + except KeyError: + plotting_properties = {**plotting_properties,**kwargs} + + # Plot data + try: + axv[axi].plot(fieldvalues,heightvalues,**plotting_properties) + except ValueError as e: + print(e,'--', time, 'not found in index?') + + # Set field label if known + try: + axv[axi].set_xlabel(args.fieldlabels[field]) + except KeyError: + pass + # Set field limits if specified + try: + axv[axi].set_xlim(args.fieldlimits[field]) + except KeyError: + pass + + for axi in axv: + axi.grid(True,which='both') + + # Set height limits if specified + if not heightlimits is None: + axv[0].set_ylim(heightlimits) + + # Add y labels + for r in range(nrows): + axv[r*ncols].set_ylabel(r'Height [m]') + + # Align labels + _align_labels(fig,axv,nrows,ncols) + + # Number sub figures as a, b, c, ... + if labelsubplots is not False: + try: + hoffset, voffset = labelsubplots + except (TypeError, ValueError): + hoffset, voffset = -0.14, -0.18 + for i,axi in enumerate(axv): + axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) + + # Add legend + if showlegend: + leg = _format_legend(axv,index=ncols-1) + + return fig,ax + + +def plot_spectrum(datasets, + fields=None, + height=None, + times=None, + fig=None,ax=None, + fieldlimits=None, + freqlimits=None, + fieldlabels={}, + labelsubplots=False, + showlegend=None, + ncols=None, + subfigsize=(4,5), + datasetkwargs={}, + **kwargs + ): + """ + Plot frequency spectrum at a given height for different datasets, + time(s) and field(s), using a subplot per time and per field. + + Note that this function does not interpolate to the requested height, + i.e., if height is not None, the specified value should be available + in all datasets. + + Usage + ===== + datasets : pandas.DataFrame or dict + Dataset(s) with spectrum data. If more than one set, + datasets should be a dictionary with entries + : dataset + fields : str, list, 'all' (or None) + Fieldname(s) corresponding to particular column(s) of + the datasets. fields can be None if input are Series. + 'all' means all fields will be plotted (in this case all + datasets should have the same fields) + height : float (or None) + Height for which frequency spectra is plotted. If datasets + have no height dimension, height does not need to be specified. + times : str, int, float, list (or None) + Time(s) for which frequency spectra are plotted, specified as + either datetime strings or numerical values (seconds, e.g., + simulation time). times can be None if all datasets combined + have no more than one time value. + fig : figure handle + Custom figure handle. Should be specified together with ax + ax : axes handle, or list or numpy ndarray with axes handles + Customand axes handle(s). + Size of ax should equal nfields * ntimes + fieldlimits : list or tuple, or dict + Value range for the various fields. If only one field is + plotted, fieldlimits can be a list or tuple. Otherwise, it + should be a dictionary with entries : fieldlimit. + Missing fieldlimits are set automatically + freqlimits : list or tuple + Frequency axis limits + fieldlabels : str or dict + Custom field labels. If only one field is plotted, fieldlabels + can be a string. Otherwise it should be a dictionary with + entries : fieldlabel + labelsubplots : bool, list or tuple + Label subplots as (a), (b), (c), ... If a list or tuple is given + their values should be the horizontal and vertical position + relative to each subaxis. + showlegend : bool (or None) + Label different plots and show legend. If None, showlegend is set + to True if legend will have more than one entry, otherwise it is + set to False. + ncols : int + Number of columns in axes grid, must be a true divisor of total + number of axes. + subfigsize : list or tuple + Standard size of subfigures + datasetkwargs : dict + Dataset-specific options that are passed on to the actual + plotting function. These options overwrite general options + specified through **kwargs. The argument should be a dictionary + with entries : {**kwargs} + **kwargs : other keyword arguments + Options that are passed on to the actual plotting function. + Note that these options should be the same for all datasets, + fields and times, and they can not be used to set dataset, + field or time specific colors, limits, etc. + Example uses include setting linestyle/width, marker, etc. + """ + + args = PlottingInput( + datasets=datasets, + fields=fields, + times=times, + fieldlimits=fieldlimits, + fieldlabels=fieldlabels, + ) + + nfields = len(args.fields) + ntimes = len(args.times) + ndatasets = len(args.datasets) + ntotal = nfields * ntimes + + # Concatenate custom and standard field labels + # (custom field labels overwrite standard fields labels if existent) + args.fieldlabels = {**standard_spectrumlabels, **args.fieldlabels} + + fig, ax, nrows, ncols = _create_subplots_if_needed( + ntotal, + ncols, + default_ncols=ntimes, + avoid_single_column=True, + sharex=True, + subfigsize=subfigsize, + wspace=0.3, + fig=fig, + ax=ax, + ) + + # Create flattened view of axes + axv = np.asarray(ax).reshape(-1) + + # Set showlegend if not specified + if showlegend is None: + if ndatasets>1: + showlegend = True + else: + showlegend = False + + # Loop over datasets, fields and times + for i, dfname in enumerate(args.datasets): + df = args.datasets[dfname] + + frequencyvalues = _get_dim_values(df,'frequency',default_idx=True) + assert(frequencyvalues is not None), 'spectrum plot needs a frequency axis' + timevalues = _get_dim_values(df,'time') + + # Create list with available fields only + available_fields = _get_available_fieldnames(df,args.fields) + + for j, field in enumerate(args.fields): + # If available_fields is [None,], fieldname is unimportant + if available_fields == [None]: + pass + # Else, check if field is available + elif not field in available_fields: + print('Warning: field "'+field+'" not available in dataset '+dfname) + continue + + for k, time in enumerate(args.times): + plotting_properties = {} + if showlegend: + plotting_properties['label'] = dfname + + # Index of axis corresponding to field j and time k + axi = j*ntimes + k + + # Axes mark up + if i==0 and ntimes>1: + axv[axi].set_title(pd.to_datetime(time).strftime('%Y-%m-%d %H%M UTC'),fontsize=16) + + # Gather label, general options and dataset-specific options + # (highest priority to dataset-specific options, then general options) + try: + plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} + except KeyError: + plotting_properties = {**plotting_properties,**kwargs} + + # Get field spectrum + slice_t = _get_slice(df,time,'time') + slice_tz = _get_slice(slice_t,height,'height') + spectrum = _get_field(slice_tz,field).values + + # Plot data + axv[axi].loglog(frequencyvalues[1:],spectrum[1:],**plotting_properties) + + # Specify field limits if specified + try: + axv[axi].set_ylim(args.fieldlimits[field]) + except KeyError: + pass + + + # Set frequency label + for c in range(ncols): + axv[ncols*(nrows-1)+c].set_xlabel('$f$ [Hz]') + + # Specify field label if specified + for r in range(nrows): + try: + axv[r*ncols].set_ylabel(args.fieldlabels[args.fields[r]]) + except KeyError: + pass + + # Align labels + _align_labels(fig,axv,nrows,ncols) + + # Set frequency limits if specified + if not freqlimits is None: + axv[0].set_xlim(freqlimits) + + # Number sub figures as a, b, c, ... + if labelsubplots is not False: + try: + hoffset, voffset = labelsubplots + except (TypeError, ValueError): + hoffset, voffset = -0.14, -0.18 + for i,axi in enumerate(axv): + axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) + + # Add legend + if showlegend: + leg = _format_legend(axv,index=ncols-1) + + return fig, ax + + +# --------------------------------------------- +# +# DEFINITION OF AUXILIARY CLASSES AND FUNCTIONS +# +# --------------------------------------------- + +class InputError(Exception): + """Exception raised for errors in the input. + + Attributes: + message -- explanation of the error + """ + + def __init__(self, message): + self.message = message + + +class PlottingInput(object): + """ + Auxiliary class to collect input data and options for plotting + functions, and to check if the inputs are consistent + """ + supported_datatypes = ( + pd.Series, + pd.DataFrame, + xr.DataArray, + xr.Dataset, + ) + + def __init__(self, datasets, fields, **argd): + # Add all arguments as class attributes + self.__dict__.update({'datasets':datasets, + 'fields':fields, + **argd}) + + # Check consistency of all attributes + self._check_consistency() + + def _check_consistency(self): + """ + Check consistency of all input data + """ + + # ---------------------- + # Check dataset argument + # ---------------------- + # If a single dataset is provided, convert to a dictionary + # under a generic key 'Dataset' + if isinstance(self.datasets, self.supported_datatypes): + self.datasets = {'Dataset': self.datasets} + for dfname,df in self.datasets.items(): + # convert dataset types here + if isinstance(df, (xr.Dataset,xr.DataArray)): + # handle xarray datatypes + self.datasets[dfname] = df.to_dataframe() + columns = self.datasets[dfname].columns + if len(columns) == 1: + # convert to pd.Series + self.datasets[dfname] = self.datasets[dfname][columns[0]] + else: + assert(isinstance(df, self.supported_datatypes)), \ + "Dataset {:s} of type {:s} not supported".format(dfname,str(type(df))) + + # ---------------------- + # Check fields argument + # ---------------------- + # If no fields are specified, check that + # - all datasets are series + # - the name of every series is either None or matches other series names + if self.fields is None: + assert(all([isinstance(self.datasets[dfname],pd.Series) for dfname in self.datasets])), \ + "'fields' argument must be specified unless all datasets are pandas Series" + series_names = set() + for dfname in self.datasets: + series_names.add(self.datasets[dfname].name) + if len(series_names)==1: + self.fields = list(series_names) + else: + raise InputError('attempting to plot multiple series with different field names') + elif isinstance(self.fields,str): + # If fields='all', retrieve fields from dataset + if self.fields=='all': + self.fields = _get_fieldnames(list(self.datasets.values())[0]) + assert(all([_get_fieldnames(df)==self.fields for df in self.datasets.values()])), \ + "The option fields = 'all' only works when all datasets have the same fields" + # If fields is a single instance, convert to a list + else: + self.fields = [self.fields,] + + # ---------------------------------- + # Check match of fields and datasets + # ---------------------------------- + # Check if all datasets have at least one of the requested fields + for dfname in self.datasets: + df = self.datasets[dfname] + if isinstance(df,pd.DataFrame): + assert(any([field in df.columns for field in self.fields])), \ + 'DataFrame '+dfname+' does not contain any of the requested fields' + elif isinstance(df,pd.Series): + if df.name is None: + assert(len(self.fields)==1), \ + 'Series must have a name if more than one fields is specified' + else: + assert(df.name in self.fields), \ + 'Series '+dfname+' does not match any of the requested fields' + + # --------------------------------- + # Check heights argument (optional) + # --------------------------------- + try: + # If no heights are specified, check that all datasets combined have + # no more than one height value + if self.heights is None: + av_heights = set() + for df in self.datasets.values(): + heightvalues = _get_dim_values(df,'height') + try: + for height in heightvalues: + av_heights.add(height) + except TypeError: + # heightvalues is None + pass + if len(av_heights)==0: + # None of the datasets have height values + self.heights = [None,] + elif len(av_heights)==1: + self.heights = list(av_heights) + else: + raise InputError("found more than one height value so 'heights' argument must be specified") + # If heights='all', retrieve heights from dataset + elif isinstance(self.heights,str) and self.heights=='all': + self.heights = _get_dim_values(list(self.datasets.values())[0],'height') + assert(all([np.allclose(_get_dim_values(df,'height'),self.heights) for df in self.datasets.values()])), \ + "The option heights = 'all' only works when all datasets have the same vertical levels" + # If heights is single instance, convert to list + elif isinstance(self.heights,(int,float)): + self.heights = [self.heights,] + except AttributeError: + pass + + # ----------------------------------- + # Check timerange argument (optional) + # ----------------------------------- + try: + if self.timerange is not None: + if self.times is not None: + print('Using specified time range',self.timerange, + 'and ignoring',self.times) + assert isinstance(self.timerange,(tuple,list)), \ + 'Need to specify timerange as (starttime,endtime)' + assert (len(self.timerange) == 2) + try: + starttime = pd.to_datetime(self.timerange[0]) + endtime = pd.to_datetime(self.timerange[1]) + except ValueError: + print('Unable to convert timerange to timestamps') + else: + # get unique times from all datasets + alltimes = [] + for df in self.datasets.values(): + alltimes += list(_get_dim_values(df,'time')) + alltimes = pd.DatetimeIndex(np.unique(alltimes)) + inrange = (alltimes >= starttime) & (alltimes <= endtime) + self.times = alltimes[inrange] + except AttributeError: + pass + + # --------------------------------- + # Check times argument (optional) + # --------------------------------- + # If times is single instance, convert to list + try: + # If no times are specified, check that all datasets combined have + # no more than one time value + if self.times is None: + av_times = set() + for df in self.datasets.values(): + timevalues = _get_dim_values(df,'time') + try: + for time in timevalues.values: + av_times.add(time) + except AttributeError: + pass + if len(av_times)==0: + # None of the datasets have time values + self.times = [None,] + elif len(av_times)==1: + self.times = list(av_times) + else: + raise InputError("found more than one time value so 'times' argument must be specified") + elif isinstance(self.times,(str,int,float,np.number,pd.Timestamp)): + self.times = [self.times,] + except AttributeError: + pass + + # ------------------------------------- + # Check fieldlimits argument (optional) + # ------------------------------------- + # If one set of fieldlimits is specified, check number of fields + # and convert to dictionary + try: + if self.fieldlimits is None: + self.fieldlimits = {} + elif isinstance(self.fieldlimits, (list, tuple)): + assert(len(self.fields)==1), 'Unclear to what field fieldlimits corresponds' + self.fieldlimits = {self.fields[0]:self.fieldlimits} + except AttributeError: + self.fieldlimits = {} + + # ------------------------------------- + # Check fieldlabels argument (optional) + # ------------------------------------- + # If one fieldlabel is specified, check number of fields + try: + if isinstance(self.fieldlabels, str): + assert(len(self.fields)==1), 'Unclear to what field fieldlabels corresponds' + self.fieldlabels = {self.fields[0]: self.fieldlabels} + except AttributeError: + self.fieldlabels = {} + + # ------------------------------------- + # Check colorscheme argument (optional) + # ------------------------------------- + # If one colorscheme is specified, check number of fields + try: + self.cmap = {} + if isinstance(self.colorschemes, str): + assert(len(self.fields)==1), 'Unclear to what field colorschemes corresponds' + self.cmap[self.fields[0]] = mpl.cm.get_cmap(self.colorschemes) + else: + # Set missing colorschemes to viridis + for field in self.fields: + if field not in self.colorschemes.keys(): + if field == 'wdir': + self.colorschemes[field] = 'twilight' + else: + self.colorschemes[field] = 'viridis' + self.cmap[field] = mpl.cm.get_cmap(self.colorschemes[field]) + except AttributeError: + pass + + # ------------------------------------- + # Check fieldorder argument (optional) + # ------------------------------------- + # Make sure fieldorder is recognized + try: + assert(self.fieldorder in ['C','F']), "Error: fieldorder '"\ + +self.fieldorder+"' not recognized, must be either 'C' or 'F'" + except AttributeError: + pass + + + def set_missing_fieldlimits(self): + """ + Set missing fieldlimits to min and max over all datasets + """ + for field in self.fields: + if field not in self.fieldlimits.keys(): + try: + self.fieldlimits[field] = [ + min([_get_field(df,field).min() for df in self.datasets.values() if _contains_field(df,field)]), + max([_get_field(df,field).max() for df in self.datasets.values() if _contains_field(df,field)]) + ] + except ValueError: + self.fieldlimits[field] = [None,None] + +def _get_dim(df,dim,default_idx=False): + """ + Search for specified dimension in dataset and return + level (referred to by either label or position) and + axis {0 or ‘index’, 1 or ‘columns’} + + If default_idx is True, return a single unnamed index + if present + """ + assert(dim in dimension_names.keys()), \ + "Dimension '"+dim+"' not supported" + + # 1. Try to find dim based on name + for name in dimension_names[dim]: + if name in df.index.names: + if debug: print("Found "+dim+" dimension in index with name '{}'".format(name)) + return name, 0 + else: + try: + if name in df.columns: + if debug: print("Found "+dim+" dimension in column with name '{}'".format(name)) + return name, 1 + except AttributeError: + # pandas Series has no columns + pass + + # 2. Look for Datetime or Timedelta index + if dim=='time': + for idx in range(len(df.index.names)): + if isinstance(df.index.get_level_values(idx),(pd.DatetimeIndex,pd.TimedeltaIndex,pd.PeriodIndex)): + if debug: print("Found "+dim+" dimension in index with level {} without a name ".format(idx)) + return idx, 0 + + # 3. If default index is True, assume that a + # single nameless index corresponds to the + # requested dimension + if (not isinstance(df.index,(pd.MultiIndex,pd.DatetimeIndex,pd.TimedeltaIndex,pd.PeriodIndex)) + and default_idx and (df.index.name is None) ): + if debug: print("Assuming nameless index corresponds to '{}' dimension".format(dim)) + return 0,0 + + # 4. Did not found requested dimension + if debug: print("Found no "+dim+" dimension") + return None, None + + +def _get_available_fieldnames(df,fieldnames): + """ + Return subset of fields available in df + """ + available_fieldnames = [] + if isinstance(df,pd.DataFrame): + for field in fieldnames: + if field in df.columns: + available_fieldnames.append(field) + # A Series only has one field, so return that field name + # (if that field is not in fields, an error would have been raised) + elif isinstance(df,pd.Series): + available_fieldnames.append(df.name) + return available_fieldnames + + +def _get_fieldnames(df): + """ + Return list of fieldnames in df + """ + if isinstance(df,pd.DataFrame): + fieldnames = list(df.columns) + # Remove any column corresponding to + # a dimension (time, height or frequency) + for dim in dimension_names.keys(): + name, axis = _get_dim(df,dim) + if axis==1: + fieldnames.remove(name) + return fieldnames + elif isinstance(df,pd.Series): + return [df.name,] + + +def _contains_field(df,fieldname): + if isinstance(df,pd.DataFrame): + return fieldname in df.columns + elif isinstance(df,pd.Series): + return (df.name is None) or (df.name==fieldname) + + +def _get_dim_values(df,dim,default_idx=False): + """ + Return values for a given dimension + """ + level, axis = _get_dim(df,dim,default_idx) + # Requested dimension is an index + if axis==0: + return df.index.get_level_values(level).unique() + # Requested dimension is a column + elif axis==1: + return df[level].unique() + # Requested dimension not available + else: + return None + + +def _get_pivot_table(df,dim,fieldnames): + """ + Return pivot table with given fieldnames as columns + """ + level, axis = _get_dim(df,dim) + # Unstack an index + if axis==0: + return df.unstack(level=level) + # Pivot about a column + elif axis==1: + return df.pivot(columns=level,values=fieldnames) + # Dimension not found, return dataframe + else: + return df + + +def _get_slice(df,key,dim): + """ + Return cross-section of dataset + """ + if key is None: + return df + + # Get dimension level and axis + level, axis = _get_dim(df,dim) + + # Requested dimension is an index + if axis==0: + if isinstance(df.index,pd.MultiIndex): + return df.xs(key,level=level) + else: + return df.loc[df.index==key] + # Requested dimension is a column + elif axis==1: + return df.loc[df[level]==key] + # Requested dimension not available, return dataframe + else: + return df + + +def _get_field(df,fieldname): + """ + Return field from dataset + """ + if isinstance(df,pd.DataFrame): + return df[fieldname] + elif isinstance(df,pd.Series): + if df.name is None or df.name==fieldname: + return df + else: + return None + + +def _get_pivoted_field(df,fieldname): + """ + Return field from pivoted dataset + """ + if isinstance(df.columns,pd.MultiIndex): + return df[fieldname] + else: + return df + + +def _create_subplots_if_needed(ntotal, + ncols=None, + default_ncols=1, + fieldorder='C', + avoid_single_column=False, + sharex=False, + sharey=False, + subfigsize=(12,3), + wspace=0.2, + hspace=0.2, + fig=None, + ax=None + ): + """ + Auxiliary function to create fig and ax + + If fig and ax are None: + - Set nrows and ncols based on ntotal and specified ncols, + accounting for fieldorder and avoid_single_column + - Create fig and ax with nrows and ncols, taking into account + sharex, sharey, subfigsize, wspace, hspace + + If fig and ax are not None: + - Try to determine nrows and ncols from ax + - Check whether size of ax corresponds to ntotal + """ + + if ax is None: + if not ncols is None: + # Use ncols if specified and appropriate + assert(ntotal%ncols==0), 'Error: Specified number of columns is not a true divisor of total number of subplots' + nrows = int(ntotal/ncols) + else: + # Defaut number of columns + ncols = default_ncols + nrows = int(ntotal/ncols) + + if fieldorder=='F': + # Swap number of rows and columns + nrows, ncols = ncols, nrows + + if avoid_single_column and ncols==1: + # Swap number of rows and columns + nrows, ncols = ncols, nrows + + # Create fig and ax with nrows and ncols + fig,ax = plt.subplots(nrows=nrows,ncols=ncols,sharex=sharex,sharey=sharey,figsize=(subfigsize[0]*ncols,subfigsize[1]*nrows)) + + # Adjust subplot spacing + fig.subplots_adjust(wspace=wspace,hspace=hspace) + + else: + # Make sure user-specified axes has appropriate size + assert(np.asarray(ax).size==ntotal), 'Specified axes does not have the right size' + + # Determine nrows and ncols in specified axes + if isinstance(ax,mpl.axes.Axes): + nrows, ncols = (1,1) + else: + try: + nrows,ncols = np.asarray(ax).shape + except ValueError: + # ax array has only one dimension + # Determine whether ax is single row or single column based + # on individual ax positions x0 and y0 + x0s = [axi.get_position().x0 for axi in ax] + y0s = [axi.get_position().y0 for axi in ax] + if all(x0==x0s[0] for x0 in x0s): + # All axis have same relative x0 position + nrows = np.asarray(ax).size + ncols = 1 + elif all(y0==y0s[0] for y0 in y0s): + # All axis have same relative y0 position + nrows = 1 + ncols = np.asarray(ax).size + else: + # More complex axes configuration, + # currently not supported + raise InputError('could not determine nrows and ncols in specified axes, complex axes configuration currently not supported') + + return fig, ax, nrows, ncols + + +def _format_legend(axv,index): + """ + Auxiliary function to format legend + + Usage + ===== + axv : numpy 1d array + Flattened array of axes + index : int + Index of the axis where to place the legend + """ + all_handles = [] + all_labels = [] + # Check each axes and add new handle + for axi in axv: + handles, labels = axi.get_legend_handles_labels() + for handle,label in zip(handles,labels): + if not label in all_labels: + all_labels.append(label) + all_handles.append(handle) + + leg = axv[index].legend(all_handles,all_labels,loc='upper left',bbox_to_anchor=(1.05,1.0),fontsize=16) + return leg + + +def _format_time_axis(fig,ax, + plot_local_time, + local_time_offset, + timelimits + ): + """ + Auxiliary function to format time axis + """ + ax[-1].xaxis_date() + if timelimits is not None: + timelimits = [pd.to_datetime(tlim) for tlim in timelimits] + hour_interval = _determine_hourlocator_interval(ax[-1],timelimits) + if plot_local_time is not False: + if plot_local_time is True: + localtimefmt = '%I %p' + else: + assert isinstance(plot_local_time,str), 'Unexpected plot_local_time format' + localtimefmt = plot_local_time + # Format first axis (local time) + ax[-1].xaxis.set_minor_locator(mdates.HourLocator(byhour=range(0,24,hour_interval))) + ax[-1].xaxis.set_minor_formatter(mdates.DateFormatter(localtimefmt)) + ax[-1].xaxis.set_major_locator(mdates.DayLocator(interval=12)) #Choose large interval so dates are not plotted + ax[-1].xaxis.set_major_formatter(mdates.DateFormatter('')) + + # Set time limits if specified + if not timelimits is None: + local_timelimits = pd.to_datetime(timelimits) + pd.to_timedelta(local_time_offset,'h') + ax[-1].set_xlim(local_timelimits) + + tstr = 'Local time' + + ax2 = [] + for axi in ax: + # Format second axis (UTC time) + ax2i = axi.twiny() + ax2i.xaxis_date() + + # Set time limits if specified + if not timelimits is None: + ax2i.set_xlim(timelimits) + else: + # Extract timelimits from main axis + local_timelimits = mdates.num2date(axi.get_xlim()) + timelimits = pd.to_datetime(local_timelimits) - pd.to_timedelta(local_time_offset,'h') + ax2i.set_xlim(timelimits) + + # Move twinned axis ticks and label from top to bottom + ax2i.xaxis.set_ticks_position("bottom") + ax2i.xaxis.set_label_position("bottom") + + # Offset the twin axis below the host + ax2i.spines["bottom"].set_position(("axes", -0.35)) + + # Turn on the frame for the twin axis, but then hide all + # but the bottom spine + ax2i.set_frame_on(True) + ax2i.patch.set_visible(False) + #for sp in ax2.spines.itervalues(): + # sp.set_visible(False) + ax2i.spines["bottom"].set_visible(True) + + ax2i.xaxis.set_minor_locator(mdates.HourLocator(byhour=range(24),interval=hour_interval)) + ax2i.xaxis.set_minor_formatter(mdates.DateFormatter('%H%M')) + ax2i.xaxis.set_major_locator(mdates.DayLocator()) + ax2i.xaxis.set_major_formatter(mdates.DateFormatter('\n%Y-%m-%d')) + ax2i.set_xlabel('UTC time') + + ax2.append(ax2i) + + if len(ax2)==1: + ax2 = ax2[0] + else: + ax2 = np.array(ax2) + fig.align_xlabels(ax2) + else: + ax[-1].xaxis.set_minor_locator(mdates.HourLocator(byhour=range(0,24,hour_interval))) + ax[-1].xaxis.set_minor_formatter(mdates.DateFormatter('%H%M')) + ax[-1].xaxis.set_major_locator(mdates.DayLocator()) + ax[-1].xaxis.set_major_formatter(mdates.DateFormatter('\n%Y-%m-%d')) + + # Set time limits if specified + if not timelimits is None: + ax[-1].set_xlim(timelimits) + + tstr = 'UTC time' + ax2 = None + + # Now, update all axes + for axi in ax: + # Make sure both major and minor axis labels are visible when they are + # at the same time + axi.xaxis.remove_overlapping_locs = False + + # Set time label + axi.set_xlabel(tstr) + + return ax2 + + +def _determine_hourlocator_interval(ax,timelimits=None): + """ + Determine hour interval based on timelimits + + If plotted time period is + - less than 36 hours: interval = 3 + - less than 72 hours: interval = 6 + - otherwise: interval = 12 + """ + # Get timelimits + if timelimits is None: + timelimits = pd.to_datetime(mdates.num2date(ax.get_xlim())) + elif isinstance(timelimits[0],str): + timelimits = pd.to_datetime(timelimits) + + # Determine time period in hours + timeperiod = (timelimits[1] - timelimits[0])/pd.to_timedelta(1,'h') + # HourLocator interval + if timeperiod < 36: + return 3 + elif timeperiod < 72: + return 6 + else: + return 12 + + +def _get_staggered_grid(x): + """ + Return staggered grid locations + + For input array size N, output array + has a size of N+1 + """ + idx = np.arange(x.size) + f = interp1d(idx,x,fill_value='extrapolate') + return f(np.arange(-0.5,x.size+0.5,1)) + + +def _align_labels(fig,ax,nrows,ncols): + """ + Align labels of a given axes grid + """ + # Align xlabels row by row + for r in range(nrows): + fig.align_xlabels(ax[r*ncols:(r+1)*ncols]) + # Align ylabels column by column + for c in range(ncols): + fig.align_ylabels(ax[c::ncols]) + + +def reference_lines(x_range, y_start, slopes, line_type='log'): + ''' + This will generate an array of y-values over a specified x-range for + the provided slopes. All lines will start from the specified + location. For now, this is only assumed useful for log-log plots. + x_range : array + values over which to plot the lines (requires 2 or more values) + y_start : float + where the lines will start in y + slopes : float or array + the slopes to be plotted (can be 1 or several) + ''' + if type(slopes)==float: + y_range = np.asarray(x_range)**slopes + shift = y_start/y_range[0] + y_range = y_range*shift + elif isinstance(slopes,(list,np.ndarray)): + y_range = np.zeros((np.shape(x_range)[0],np.shape(slopes)[0])) + for ss,slope in enumerate(slopes): + y_range[:,ss] = np.asarray(x_range)**slope + shift = y_start/y_range[0,ss] + y_range[:,ss] = y_range[:,ss]*shift + return(y_range) + + +class TaylorDiagram(object): + """ + Taylor diagram. + + Plot model standard deviation and correlation to reference (data) + sample in a single-quadrant polar plot, with r=stddev and + theta=arccos(correlation). + + Based on code from Yannick Copin + Downloaded from https://gist.github.com/ycopin/3342888 on 2020-06-19 + """ + + def __init__(self, refstd, + fig=None, rect=111, label='_', srange=(0, 1.5), extend=False, + normalize=False, + corrticks=[0, 0.2, 0.4, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1], + minorcorrticks=None, + stdevticks=None, + labelsize=None): + """ + Set up Taylor diagram axes, i.e. single quadrant polar + plot, using `mpl_toolkits.axisartist.floating_axes`. + + Usage + ===== + refstd: np.ndarray + Reference standard deviation to be compared to + fig: plt.Figure, optional + Input figure or None to create a new figure + rect: 3-digit integer + Subplot position, described by: nrows, ncols, index + label: str, optional + Legend label for reference point + srange: tuple, optional + Stdev axis limits, in units of *refstd* + extend: bool, optional + Extend diagram to negative correlations + normalize: bool, optional + Normalize stdev axis by `refstd` + corrticks: list-like, optional + Specify ticks positions on azimuthal correlation axis + minorcorrticks: list-like, optional + Specify minor tick positions on azimuthal correlation axis + stdevticks: int or list-like, optional + Specify stdev axis grid locator based on MaxNLocator (with + integer input) or FixedLocator (with list-like input) + labelsize: int or str, optional + Font size (e.g., 16 or 'x-large') for all axes labels + """ + + from matplotlib.projections import PolarAxes + from mpl_toolkits.axisartist import floating_axes + from mpl_toolkits.axisartist import grid_finder + + self.refstd = refstd # Reference standard deviation + self.normalize = normalize + + tr = PolarAxes.PolarTransform() + + # Correlation labels + if minorcorrticks is None: + rlocs = np.array(corrticks) + else: + rlocs = np.array(sorted(list(corrticks) + list(minorcorrticks))) + if extend: + # Diagram extended to negative correlations + self.tmax = np.pi + rlocs = np.concatenate((-rlocs[:0:-1], rlocs)) + else: + # Diagram limited to positive correlations + self.tmax = np.pi/2 + if minorcorrticks is None: + rlocstrs = [str(rloc) for rloc in rlocs] + else: + rlocstrs = [str(rloc) if abs(rloc) in corrticks else '' + for rloc in rlocs] + tlocs = np.arccos(rlocs) # Conversion to polar angles + gl1 = grid_finder.FixedLocator(tlocs) # Positions + tf1 = grid_finder.DictFormatter(dict(zip(tlocs, rlocstrs))) + + # Stdev labels + if isinstance(stdevticks, int): + gl2 = grid_finder.MaxNLocator(stdevticks) + elif hasattr(stdevticks, '__iter__'): + gl2 = grid_finder.FixedLocator(stdevticks) + else: + gl2 = None + + # Standard deviation axis extent (in units of reference stddev) + self.smin, self.smax = srange + if not normalize: + self.smin *= self.refstd + self.smax *= self.refstd + + ghelper = floating_axes.GridHelperCurveLinear( + tr, + extremes=(0, self.tmax, self.smin, self.smax), + grid_locator1=gl1, + grid_locator2=gl2, + tick_formatter1=tf1, + #tick_formatter2=tf2 + ) + + if fig is None: + fig = plt.figure() + + ax = floating_axes.FloatingSubplot(fig, rect, grid_helper=ghelper) + fig.add_subplot(ax) + + # Adjust axes + # - angle axis + ax.axis["top"].set_axis_direction("bottom") + ax.axis["top"].toggle(ticklabels=True, label=True) + ax.axis["top"].major_ticklabels.set_axis_direction("top") + ax.axis["top"].label.set_axis_direction("top") + ax.axis["top"].label.set_text("Correlation") + + # - "x" axis + ax.axis["left"].set_axis_direction("bottom") + if normalize: + ax.axis["left"].label.set_text("Normalized standard deviation") + else: + ax.axis["left"].label.set_text("Standard deviation") + + # - "y" axis + ax.axis["right"].set_axis_direction("top") # "Y-axis" + ax.axis["right"].toggle(ticklabels=True) + ax.axis["right"].major_ticklabels.set_axis_direction( + "bottom" if extend else "left") + + # Set label sizes + if labelsize is not None: + ax.axis["top"].label.set_fontsize(labelsize) + ax.axis["left"].label.set_fontsize(labelsize) + ax.axis["right"].label.set_fontsize(labelsize) + ax.axis["top"].major_ticklabels.set_fontsize(labelsize) + ax.axis["left"].major_ticklabels.set_fontsize(labelsize) + ax.axis["right"].major_ticklabels.set_fontsize(labelsize) + + if self.smin: + # get rid of cluster of labels at origin + ax.axis["bottom"].toggle(ticklabels=False, label=False) + else: + ax.axis["bottom"].set_visible(False) # Unused + + self._ax = ax # Graphical axes + self.ax = ax.get_aux_axes(tr) # Polar coordinates + + # Add reference point and stddev contour + t = np.linspace(0, self.tmax) + r = np.ones_like(t) + if self.normalize: + l, = self.ax.plot([0], [1], 'k*', ls='', ms=10, label=label) + else: + l, = self.ax.plot([0], self.refstd, 'k*', ls='', ms=10, label=label) + r *= refstd + self.ax.plot(t, r, 'k--', label='_') + + # Collect sample points for latter use (e.g. legend) + self.samplePoints = [l] + + def set_ref(self, refstd): + """ + Update the reference standard deviation value + + Useful for cases in which datasets with different reference + values (e.g., originating from different reference heights) + are to be overlaid on the same diagram. + """ + self.refstd = refstd + + def add_sample(self, stddev, corrcoef, norm=None, *args, **kwargs): + """ + Add sample (*stddev*, *corrcoeff*) to the Taylor + diagram. *args* and *kwargs* are directly propagated to the + `Figure.plot` command. + + `norm` may be specified to override the default normalization + value if TaylorDiagram was initialized with normalize=True + """ + if (corrcoef < 0) and (self.tmax == np.pi/2): + print('Note: ({:g},{:g}) not shown for R2 < 0, set extend=True'.format(stddev,corrcoef)) + return None + + if self.normalize: + if norm is None: + norm = self.refstd + elif norm is False: + norm = 1 + stddev /= norm + + l, = self.ax.plot(np.arccos(corrcoef), stddev, + *args, **kwargs) # (theta, radius) + self.samplePoints.append(l) + + return l + + def add_grid(self, *args, **kwargs): + """Add a grid.""" + + self._ax.grid(*args, **kwargs) + + def add_contours(self, levels=5, scale=1.0, **kwargs): + """ + Add constant centered RMS difference contours, defined by *levels*. + """ + + rs, ts = np.meshgrid(np.linspace(self.smin, self.smax), + np.linspace(0, self.tmax)) + # Compute centered RMS difference + if self.normalize: + # - normalized refstd == 1 + # - rs values were previously normalized in __init__ + # - premultiply with (scale==refstd) to get correct rms diff + rms = scale * np.sqrt(1 + rs**2 - 2*rs*np.cos(ts)) + else: + rms = np.sqrt(self.refstd**2 + rs**2 - 2*self.refstd*rs*np.cos(ts)) + + contours = self.ax.contour(ts, rs, rms, levels, **kwargs) + + return contours + + def set_xlabel(self, label, fontsize=None): + """ + Set the label for the standard deviation axis + """ + self._ax.axis["left"].label.set_text(label) + if fontsize is not None: + self._ax.axis["left"].label.set_fontsize(fontsize) + + def set_alabel(self, label, fontsize=None): + """ + Set the label for the azimuthal axis + """ + self._ax.axis["top"].label.set_text(label) + if fontsize is not None: + self._ax.axis["top"].label.set_fontsize(fontsize) + + def set_title(self, label, **kwargs): + """ + Set the title for the axes + """ + self._ax.set_title(label, **kwargs) + + From 368aac99d7a57966054556fc51665b87808180f2 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Mon, 26 Apr 2021 18:28:49 -0600 Subject: [PATCH 070/145] Alias mmctools.plotting to windtools.plotting, rerun plotting notebook to verify --- example_plotting.ipynb | 60 +- mmctools/plotting.py | 2114 --------------------------------- mmctools/plotting/__init__.py | 2 + 3 files changed, 32 insertions(+), 2144 deletions(-) delete mode 100644 mmctools/plotting.py create mode 100644 mmctools/plotting/__init__.py diff --git a/example_plotting.ipynb b/example_plotting.ipynb index 9cbd29d..d23f334 100644 --- a/example_plotting.ipynb +++ b/example_plotting.ipynb @@ -24,17 +24,24 @@ "import pandas as pd" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: the following cell should not be needed if a `pip install [-e]` was performed" + ] + }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "#Make sure a2e-mmc repositories are in the pythonpath\n", - "a2epath = '/home/equon/a2e-mmc'\n", - "import sys\n", - "if not a2epath in sys.path:\n", - " sys.path.append(a2epath)" + "# #Make sure a2e-mmc repositories are in the pythonpath\n", + "# a2epath = '/home/equon/a2e-mmc'\n", + "# import sys\n", + "# if not a2epath in sys.path:\n", + "# sys.path.append(a2epath)" ] }, { @@ -85,7 +92,7 @@ "metadata": {}, "outputs": [], "source": [ - "datadir = '/Users/equon/a2e-mmc/assessment/datasets/SWiFT/data'\n", + "datadir = '/home/equon/a2e-mmc/assessment/datasets/SWiFT/data'\n", "TTUdata = 'TTU_tilt_corrected_20131108-09.csv'" ] }, @@ -119,9 +126,8 @@ " u\n", " v\n", " w\n", - " Ts\n", - " T\n", - " RH\n", + " t\n", + " ts\n", " p\n", " \n", " \n", @@ -133,7 +139,6 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", @@ -143,9 +148,8 @@ " -0.138929\n", " 2.637817\n", " 0.074016\n", - " 289.410000\n", " 284.794\n", - " 26.186\n", + " 289.410000\n", " 908.547754\n", " \n", " \n", @@ -153,9 +157,8 @@ " -0.601111\n", " 2.783204\n", " 0.487330\n", - " 290.979994\n", " 284.932\n", - " 25.810\n", + " 290.979994\n", " 908.723508\n", " \n", " \n", @@ -163,9 +166,8 @@ " 0.416792\n", " 4.043940\n", " 0.295800\n", - " 287.520000\n", " 285.166\n", - " 25.380\n", + " 287.520000\n", " 908.215548\n", " \n", " \n", @@ -173,9 +175,8 @@ " -0.276479\n", " 5.227110\n", " -0.418065\n", - " 287.250000\n", " 285.298\n", - " 25.264\n", + " 287.250000\n", " 907.611414\n", " \n", " \n", @@ -183,9 +184,8 @@ " 0.034364\n", " 5.908367\n", " -0.173836\n", - " 287.610000\n", " 285.414\n", - " 24.934\n", + " 287.610000\n", " 907.307654\n", " \n", " \n", @@ -193,13 +193,13 @@ "" ], "text/plain": [ - " u v w Ts T RH \\\n", - "datetime height \n", - "2013-11-08 0.9 -0.138929 2.637817 0.074016 289.410000 284.794 26.186 \n", - " 2.4 -0.601111 2.783204 0.487330 290.979994 284.932 25.810 \n", - " 4.0 0.416792 4.043940 0.295800 287.520000 285.166 25.380 \n", - " 10.1 -0.276479 5.227110 -0.418065 287.250000 285.298 25.264 \n", - " 16.8 0.034364 5.908367 -0.173836 287.610000 285.414 24.934 \n", + " u v w t ts \\\n", + "datetime height \n", + "2013-11-08 0.9 -0.138929 2.637817 0.074016 284.794 289.410000 \n", + " 2.4 -0.601111 2.783204 0.487330 284.932 290.979994 \n", + " 4.0 0.416792 4.043940 0.295800 285.166 287.520000 \n", + " 10.1 -0.276479 5.227110 -0.418065 285.298 287.250000 \n", + " 16.8 0.034364 5.908367 -0.173836 285.414 287.610000 \n", "\n", " p \n", "datetime height \n", @@ -235,7 +235,7 @@ "source": [ "# Calculate wind speed and direction\n", "df['wspd'], df['wdir'] = calc_wind(df)\n", - "df['theta'] = theta(df['T'],df['p'])" + "df['theta'] = theta(df['t'],df['p'])" ] }, { @@ -483,9 +483,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.1" + "version": "3.7.6" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/mmctools/plotting.py b/mmctools/plotting.py deleted file mode 100644 index 884c612..0000000 --- a/mmctools/plotting.py +++ /dev/null @@ -1,2114 +0,0 @@ -""" -Library of standardized plotting functions for basic plot formats -""" -import numpy as np -import matplotlib as mpl -import matplotlib.pyplot as plt -import matplotlib.dates as mdates -import pandas as pd -import xarray as xr -from scipy.interpolate import interp1d -from scipy.signal import welch - -# Standard field labels -# - default: e.g., "Km/s" -# - all superscript: e.g., "K m s^{-1}" -fieldlabels_default_units = { - 'wspd': r'Wind speed [m/s]', - 'wdir': r'Wind direction [$^\circ$]', - 'u': r'u [m/s]', - 'v': r'v [m/s]', - 'w': r'Vertical wind speed [m/s]', - 'theta': r'$\theta$ [K]', - 'thetav': r'$\theta_v$ [K]', - 'uu': r'$\langle u^\prime u^\prime \rangle \;[\mathrm{m^2/s^2}]$', - 'vv': r'$\langle v^\prime v^\prime \rangle \;[\mathrm{m^2/s^2}]$', - 'ww': r'$\langle w^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', - 'uv': r'$\langle u^\prime v^\prime \rangle \;[\mathrm{m^2/s^2}]$', - 'uw': r'$\langle u^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', - 'vw': r'$\langle v^\prime w^\prime \rangle \;[\mathrm{m^2/s^2}]$', - 'tw': r'$\langle w^\prime \theta^\prime \rangle \;[\mathrm{Km/s}]$', - 'TI': r'TI $[-]$', - 'TKE': r'TKE $[\mathrm{m^2/s^2}]$', -} -fieldlabels_superscript_units = { - 'wspd': r'Wind speed [m s$^{-1}$]', - 'wdir': r'Wind direction [$^\circ$]', - 'u': r'u [m s$^{-1}$]', - 'v': r'v [m s$^{-1}$]', - 'w': r'Vertical wind speed [m s$^{-1}$]', - 'theta': r'$\theta$ [K]', - 'thetav': r'$\theta_v$ [K]', - 'uu': r'$\langle u^\prime u^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', - 'vv': r'$\langle v^\prime v^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', - 'ww': r'$\langle w^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', - 'uv': r'$\langle u^\prime v^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', - 'uw': r'$\langle u^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', - 'vw': r'$\langle v^\prime w^\prime \rangle \;[\mathrm{m^2 s^{-2}}]$', - 'tw': r'$\langle w^\prime \theta^\prime \rangle \;[\mathrm{K m s^{-1}}]$', - 'TI': r'TI $[-]$', - 'TKE': r'TKE $[\mathrm{m^2 s^{-2}}]$', -} - -# Standard field labels for frequency spectra -spectrumlabels_default_units = { - 'u': r'$E_{uu}\;[\mathrm{m^2/s}]$', - 'v': r'$E_{vv}\;[\mathrm{m^2/s}]$', - 'w': r'$E_{ww}\;[\mathrm{m^2/s}]$', - 'theta': r'$E_{\theta\theta}\;[\mathrm{K^2 s}]$', - 'thetav': r'$E_{\theta\theta}\;[\mathrm{K^2 s}]$', - 'wspd': r'$E_{UU}\;[\mathrm{m^2/s}]$', -} -spectrumlabels_superscript_units = { - 'u': r'$E_{uu}\;[\mathrm{m^2\;s^{-1}}]$', - 'v': r'$E_{vv}\;[\mathrm{m^2\;s^{-1}}]$', - 'w': r'$E_{ww}\;[\mathrm{m^2\;s^{-1}}]$', - 'theta': r'$E_{\theta\theta}\;[\mathrm{K^2\;s}]$', - 'thetav': r'$E_{\theta\theta}\;[\mathrm{K^2\;s}]$', - 'wspd': r'$E_{UU}\;[\mathrm{m^2\;s^{-1}}]$', -} - -# Default settings -default_colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] -standard_fieldlabels = fieldlabels_default_units -standard_spectrumlabels = spectrumlabels_default_units - -# Supported dimensions and associated names -dimension_names = { - 'time': ['datetime','time','Time','t'], - 'height': ['height','heights','z'], - 'frequency': ['frequency','f',] -} - -# Show debug information -debug = False - -def plot_timeheight(datasets, - fields=None, - fig=None,ax=None, - colorschemes={}, - fieldlimits=None, - heightlimits=None, - timelimits=None, - fieldlabels={}, - labelsubplots=False, - showcolorbars=True, - fieldorder='C', - ncols=1, - subfigsize=(12,4), - plot_local_time=False, - local_time_offset=0, - datasetkwargs={}, - **kwargs - ): - """ - Plot time-height contours for different datasets and fields - - Usage - ===== - datasets : pandas.DataFrame or dict - Dataset(s). If more than one set, datasets should - be a dictionary with entries : dataset - fields : str, list, 'all' (or None) - Fieldname(s) corresponding to particular column(s) of - the datasets. fields can be None if input are MultiIndex Series. - 'all' means all fields will be plotted (in this case all - datasets should have the same fields) - fig : figure handle - Custom figure handle. Should be specified together with ax - ax : axes handle, or list or numpy ndarray with axes handles - Customand axes handle(s). - Size of ax should equal ndatasets*nfields - colorschemes : str or dict - Name of colorschemes. If only one field is plotted, colorschemes - can be a string. Otherwise, it should be a dictionary with - entries : name_of_colorschemes - Missing colorschemess are set to 'viridis' - fieldlimits : list or tuple, or dict - Value range for the various fields. If only one field is - plotted, fieldlimits can be a list or tuple. Otherwise, it - should be a dictionary with entries : fieldlimit. - Missing fieldlimits are set automatically - heightlimits : list or tuple - Height axis limits - timelimits : list or tuple - Time axis limits - fieldlabels : str or dict - Custom field labels. If only one field is plotted, fieldlabels - can be a string. Otherwise it should be a dictionary with - entries : fieldlabel - labelsubplots : bool, list or tuple - Label subplots as (a), (b), (c), ... If a list or tuple is given - their values should be the horizontal and vertical position - relative to each subaxis. - showcolorbars : bool - Show colorbar per subplot - fieldorder : 'C' or 'F' - Index ordering for assigning fields and datasets to axes grid - (row by row). Fields is considered the first axis, so 'C' means - fields change slowest, 'F' means fields change fastest. - ncols : int - Number of columns in axes grid, must be a true divisor of total - number of axes. - subfigsize : list or tuple - Standard size of subfigures - plot_local_time : bool or str - Plot dual x axes with both UTC time and local time. If a str is - provided, then plot_local_time is assumed to be True and the str - is used as the datetime format. - local_time_offset : float - Local time offset from UTC - datasetkwargs : dict - Dataset-specific options that are passed on to the actual - plotting function. These options overwrite general options - specified through **kwargs. The argument should be a dictionary - with entries : {**kwargs} - **kwargs : other keyword arguments - Options that are passed on to the actual plotting function. - Note that these options should be the same for all datasets and - fields and can not be used to set dataset or field specific - limits, colorschemess, norms, etc. - Example uses include setting shading, rasterized, etc. - """ - - args = PlottingInput( - datasets=datasets, - fields=fields, - fieldlimits=fieldlimits, - fieldlabels=fieldlabels, - colorschemes=colorschemes, - fieldorder=fieldorder - ) - args.set_missing_fieldlimits() - - nfields = len(args.fields) - ndatasets = len(args.datasets) - ntotal = nfields * ndatasets - - # Concatenate custom and standard field labels - # (custom field labels overwrite standard fields labels if existent) - args.fieldlabels = {**standard_fieldlabels, **args.fieldlabels} - - fig, ax, nrows, ncols = _create_subplots_if_needed( - ntotal, - ncols, - sharex=True, - sharey=True, - subfigsize=subfigsize, - hspace=0.2, - fig=fig, - ax=ax - ) - - # Create flattened view of axes - axv = np.asarray(ax).reshape(-1) - - # Initialise list of colorbars - cbars = [] - - # Loop over datasets, fields and times - for i, dfname in enumerate(args.datasets): - df = args.datasets[dfname] - - heightvalues = _get_dim_values(df,'height') - timevalues = _get_dim_values(df,'time') - assert(heightvalues is not None), 'timeheight plot needs a height axis' - assert(timevalues is not None), 'timeheight plot needs a time axis' - - if isinstance(timevalues, pd.DatetimeIndex): - # If plot local time, shift timevalues - if plot_local_time is not False: - timevalues = timevalues + pd.to_timedelta(local_time_offset,'h') - - # Convert to days since 0001-01-01 00:00 UTC, plus one - numerical_timevalues = mdates.date2num(timevalues.values) - else: - if isinstance(timevalues, pd.TimedeltaIndex): - timevalues = timevalues.total_seconds() - - # Timevalues is already a numerical array - numerical_timevalues = timevalues - - # Create time-height mesh grid - tst = _get_staggered_grid(numerical_timevalues) - zst = _get_staggered_grid(heightvalues) - Ts,Zs = np.meshgrid(tst,zst,indexing='xy') - - # Create list with available fields only - available_fields = _get_available_fieldnames(df,args.fields) - - # Pivot all fields in a dataset at once - df_pivot = _get_pivot_table(df,'height',available_fields) - - for j, field in enumerate(args.fields): - # If available_fields is [None,], fieldname is unimportant - if available_fields == [None]: - pass - # Else, check if field is available - elif not field in available_fields: - print('Warning: field "'+field+'" not available in dataset '+dfname) - continue - - # Store plotting options in dictionary - plotting_properties = { - 'vmin': args.fieldlimits[field][0], - 'vmax': args.fieldlimits[field][1], - 'cmap': args.cmap[field] - } - - # Index of axis corresponding to dataset i and field j - if args.fieldorder=='C': - axi = i*nfields + j - else: - axi = j*ndatasets + i - - # Extract data from dataframe - fieldvalues = _get_pivoted_field(df_pivot,field) - - # Gather label, color, general options and dataset-specific options - # (highest priority to dataset-specific options, then general options) - try: - plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} - except KeyError: - plotting_properties = {**plotting_properties,**kwargs} - - # Plot data - im = axv[axi].pcolormesh(Ts,Zs,fieldvalues.T,**plotting_properties) - - # Colorbar mark up - if showcolorbars: - cbar = fig.colorbar(im,ax=axv[axi],shrink=1.0) - # Set field label if known - try: - cbar.set_label(args.fieldlabels[field]) - except KeyError: - pass - # Save colorbar - cbars.append(cbar) - - # Set title if more than one dataset - if ndatasets>1: - axv[axi].set_title(dfname,fontsize=16) - - - # Format time axis - if isinstance(timevalues, (pd.DatetimeIndex, pd.TimedeltaIndex)): - ax2 = _format_time_axis(fig,axv[(nrows-1)*ncols:],plot_local_time,local_time_offset,timelimits) - else: - ax2 = None - # Set time limits if specified - if not timelimits is None: - axv[-1].set_xlim(timelimits) - # Set time label - for axi in axv[(nrows-1)*ncols:]: - axi.set_xlabel('time [s]') - - if not heightlimits is None: - axv[-1].set_ylim(heightlimits) - - # Add y labels - for r in range(nrows): - axv[r*ncols].set_ylabel(r'Height [m]') - - # Align time, height and color labels - _align_labels(fig,axv,nrows,ncols) - if showcolorbars: - _align_labels(fig,[cb.ax for cb in cbars],nrows,ncols) - - # Number sub figures as a, b, c, ... - if labelsubplots is not False: - try: - hoffset, voffset = labelsubplots - except (TypeError, ValueError): - hoffset, voffset = -0.14, 1.0 - for i,axi in enumerate(axv): - axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) - - # Return cbar instead of array if ntotal==1 - if len(cbars)==1: - cbars=cbars[0] - - if (plot_local_time is not False) and ax2 is not None: - return fig, ax, ax2, cbars - else: - return fig, ax, cbars - - -def plot_timehistory_at_height(datasets, - fields=None, - heights=None, - extrapolate=True, - fig=None,ax=None, - fieldlimits=None, - timelimits=None, - fieldlabels={}, - cmap=None, - stack_by_datasets=None, - labelsubplots=False, - showlegend=None, - ncols=1, - subfigsize=(12,3), - plot_local_time=False, - local_time_offset=0, - datasetkwargs={}, - **kwargs - ): - """ - Plot time history at specified height(s) for various dataset(s) - and/or field(s). - - By default, data for multiple datasets or multiple heights are - stacked in a single subplot. When multiple datasets and multiple - heights are specified together, heights are stacked in a subplot - per field and per dataset. - - Usage - ===== - datasets : pandas.DataFrame or dict - Dataset(s). If more than one set, datasets should - be a dictionary with entries : dataset - fields : str, list, 'all' (or None) - Fieldname(s) corresponding to particular column(s) of - the datasets. fields can be None if input are Series. - 'all' means all fields will be plotted (in this case all - datasets should have the same fields) - heights : float, list, 'all' (or None) - Height(s) for which time history is plotted. heights can be - None if all datasets combined have no more than one height - value. 'all' means the time history for all heights in the - datasets will be plotted (in this case all datasets should - have the same heights) - extrapolate : bool - If false, then output height(s) outside the data range will - not be plotted; default is true for backwards compatibility - fig : figure handle - Custom figure handle. Should be specified together with ax - ax : axes handle, or list or numpy ndarray with axes handles - Customand axes handle(s). - Size of ax should equal nfields * (ndatasets or nheights) - fieldlimits : list or tuple, or dict - Value range for the various fields. If only one field is - plotted, fieldlimits can be a list or tuple. Otherwise, it - should be a dictionary with entries : fieldlimit. - Missing fieldlimits are set automatically - timelimits : list or tuple - Time axis limits - fieldlabels : str or dict - Custom field labels. If only one field is plotted, fieldlabels - can be a string. Otherwise it should be a dictionary with - entries : fieldlabel - cmap : str - Colormap used when stacking heights - stack_by_datasets : bool (or None) - Flag to specify what is plotted ("stacked") together per subfigure. - If True, stack datasets together, otherwise stack by heights. If - None, stack_by_datasets will be set based on the number of heights - and datasets. - labelsubplots : bool, list or tuple - Label subplots as (a), (b), (c), ... If a list or tuple is given - their values should be the horizontal and vertical position - relative to each subaxis. - showlegend : bool (or None) - Label different plots and show legend. If None, showlegend is set - to True if legend will have more than one entry, otherwise it is - set to False. - ncols : int - Number of columns in axes grid, must be a true divisor of total - number of axes. - subfigsize : list or tuple - Standard size of subfigures - plot_local_time : bool or str - Plot dual x axes with both UTC time and local time. If a str is - provided, then plot_local_time is assumed to be True and the str - is used as the datetime format. - local_time_offset : float - Local time offset from UTC - datasetkwargs : dict - Dataset-specific options that are passed on to the actual - plotting function. These options overwrite general options - specified through **kwargs. The argument should be a dictionary - with entries : {**kwargs} - **kwargs : other keyword arguments - Options that are passed on to the actual plotting function. - Note that these options should be the same for all datasets, - fields and heights, and they can not be used to set dataset, - field or height specific colors, limits, etc. - Example uses include setting linestyle/width, marker, etc. - """ - # Avoid FutureWarning concerning the use of an implicitly registered - # datetime converter for a matplotlib plotting method. The converter - # was registered by pandas on import. Future versions of pandas will - # require explicit registration of matplotlib converters, as done here. - from pandas.plotting import register_matplotlib_converters - register_matplotlib_converters() - - args = PlottingInput( - datasets=datasets, - fields=fields, - heights=heights, - fieldlimits=fieldlimits, - fieldlabels=fieldlabels, - ) - - nfields = len(args.fields) - nheights = len(args.heights) - ndatasets = len(args.datasets) - - # Concatenate custom and standard field labels - # (custom field labels overwrite standard fields labels if existent) - args.fieldlabels = {**standard_fieldlabels, **args.fieldlabels} - - # Set up subplot grid - if stack_by_datasets is None: - if nheights>1: - stack_by_datasets = False - else: - stack_by_datasets = True - - if stack_by_datasets: - ntotal = nfields*nheights - else: - ntotal = nfields*ndatasets - - fig, ax, nrows, ncols = _create_subplots_if_needed( - ntotal, - ncols, - sharex=True, - subfigsize=subfigsize, - hspace=0.2, - fig=fig, - ax=ax - ) - - # Create flattened view of axes - axv = np.asarray(ax).reshape(-1) - - # Set showlegend if not specified - if showlegend is None: - if (stack_by_datasets and ndatasets>1) or (not stack_by_datasets and nheights>1): - showlegend = True - else: - showlegend = False - - # Loop over datasets and fields - for i,dfname in enumerate(args.datasets): - df = args.datasets[dfname] - timevalues = _get_dim_values(df,'time',default_idx=True) - assert(timevalues is not None), 'timehistory plot needs a time axis' - heightvalues = _get_dim_values(df,'height') - - if isinstance(timevalues, pd.TimedeltaIndex): - timevalues = timevalues.total_seconds() - - # If plot local time, shift timevalues - if (plot_local_time is not False) and \ - isinstance(timevalues, (pd.DatetimeIndex, pd.TimedeltaIndex)): - timevalues = timevalues + pd.to_timedelta(local_time_offset,'h') - - # Create list with available fields only - available_fields = _get_available_fieldnames(df,args.fields) - - # If any of the requested heights is not available, - # pivot the dataframe to allow interpolation. - # Pivot all fields in a dataset at once to reduce computation time - if (not heightvalues is None) and (not all([h in heightvalues for h in args.heights])): - df_pivot = _get_pivot_table(df,'height',available_fields) - pivoted = True - fill_value = 'extrapolate' if extrapolate else np.nan - if debug: print('Pivoting '+dfname) - else: - pivoted = False - - for j, field in enumerate(args.fields): - # If available_fields is [None,], fieldname is unimportant - if available_fields == [None]: - pass - # Else, check if field is available - elif not field in available_fields: - print('Warning: field "'+field+'" not available in dataset '+dfname) - continue - - for k, height in enumerate(args.heights): - # Check if height is outside of data range - if (heightvalues is not None) and \ - ((height > np.max(heightvalues)) or (height < np.min(heightvalues))): - if extrapolate: - if debug: - print('Extrapolating field "'+field+'" at z='+str(height)+' in dataset '+dfname) - else: - print('Warning: field "'+field+'" not available at z='+str(height)+' in dataset '+dfname) - continue - - # Store plotting options in dictionary - # Set default linestyle to '-' and no markers - plotting_properties = { - 'linestyle':'-', - 'marker':None, - } - - # Axis order, label and title depend on value of stack_by_datasets - if stack_by_datasets: - # Index of axis corresponding to field j and height k - axi = k*nfields + j - - # Use datasetname as label - if showlegend: - plotting_properties['label'] = dfname - - # Set title if multiple heights are compared - if nheights>1: - axv[axi].set_title('z = {:.1f} m'.format(height),fontsize=16) - - # Set colors - plotting_properties['color'] = default_colors[i % len(default_colors)] - else: - # Index of axis corresponding to field j and dataset i - axi = i*nfields + j - - # Use height as label - if showlegend: - plotting_properties['label'] = 'z = {:.1f} m'.format(height) - - # Set title if multiple datasets are compared - if ndatasets>1: - axv[axi].set_title(dfname,fontsize=16) - - # Set colors - if cmap is not None: - cmap = mpl.cm.get_cmap(cmap) - plotting_properties['color'] = cmap(k/(nheights-1)) - else: - plotting_properties['color'] = default_colors[k % len(default_colors)] - - # Extract data from dataframe - if pivoted: - signal = interp1d(heightvalues,_get_pivoted_field(df_pivot,field).values,axis=-1,fill_value=fill_value)(height) - else: - slice_z = _get_slice(df,height,'height') - signal = _get_field(slice_z,field).values - - # Gather label, color, general options and dataset-specific options - # (highest priority to dataset-specific options, then general options) - try: - plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} - except KeyError: - plotting_properties = {**plotting_properties,**kwargs} - - # Plot data - axv[axi].plot(timevalues,signal,**plotting_properties) - - # Set field label if known - try: - axv[axi].set_ylabel(args.fieldlabels[field]) - except KeyError: - pass - # Set field limits if specified - try: - axv[axi].set_ylim(args.fieldlimits[field]) - except KeyError: - pass - - # Set axis grid - for axi in axv: - axi.xaxis.grid(True,which='both') - axi.yaxis.grid(True) - - # Format time axis - if isinstance(timevalues, (pd.DatetimeIndex, pd.TimedeltaIndex)): - ax2 = _format_time_axis(fig,axv[(nrows-1)*ncols:],plot_local_time,local_time_offset,timelimits) - else: - ax2 = None - # Set time limits if specified - if not timelimits is None: - axv[-1].set_xlim(timelimits) - # Set time label - for axi in axv[(nrows-1)*ncols:]: - axi.set_xlabel('time [s]') - - # Number sub figures as a, b, c, ... - if labelsubplots is not False: - try: - hoffset, voffset = labelsubplots - except (TypeError, ValueError): - hoffset, voffset = -0.14, 1.0 - for i,axi in enumerate(axv): - axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) - - # Add legend - if showlegend: - leg = _format_legend(axv,index=ncols-1) - - # Align labels - _align_labels(fig,axv,nrows,ncols) - - if (plot_local_time is not False) and ax2 is not None: - return fig, ax, ax2 - else: - return fig, ax - - -def plot_profile(datasets, - fields=None, - times=None, - timerange=None, - fig=None,ax=None, - fieldlimits=None, - heightlimits=None, - fieldlabels={}, - cmap=None, - stack_by_datasets=None, - labelsubplots=False, - showlegend=None, - fieldorder='C', - ncols=None, - subfigsize=(4,5), - plot_local_time=False, - local_time_offset=0, - datasetkwargs={}, - **kwargs - ): - """ - Plot vertical profile at specified time(s) for various dataset(s) - and/or field(s). - - By default, data for multiple datasets or multiple times are - stacked in a single subplot. When multiple datasets and multiple - times are specified together, times are stacked in a subplot - per field and per dataset. - - Usage - ===== - datasets : pandas.DataFrame or dict - Dataset(s). If more than one set, datasets should - be a dictionary with entries : dataset - fields : str, list, 'all' (or None) - Fieldname(s) corresponding to particular column(s) of - the datasets. fields can be None if input are Series. - 'all' means all fields will be plotted (in this case all - datasets should have the same fields) - times : str, int, float, list (or None) - Time(s) for which vertical profiles are plotted, specified as - either datetime strings or numerical values (seconds, e.g., - simulation time). times can be None if all datasets combined - have no more than one time value, or if timerange is specified. - timerange : tuple or list - Start and end times (inclusive) between which all times are - plotted. If cmap is None, then it will automatically be set to - viridis by default. This overrides times when specified. - fig : figure handle - Custom figure handle. Should be specified together with ax - ax : axes handle, or list or numpy ndarray with axes handles - Customand axes handle(s). - Size of ax should equal nfields * (ndatasets or ntimes) - fieldlimits : list or tuple, or dict - Value range for the various fields. If only one field is - plotted, fieldlimits can be a list or tuple. Otherwise, it - should be a dictionary with entries : fieldlimit. - Missing fieldlimits are set automatically - heightlimits : list or tuple - Height axis limits - fieldlabels : str or dict - Custom field labels. If only one field is plotted, fieldlabels - can be a string. Otherwise it should be a dictionary with - entries : fieldlabel - cmap : str - Colormap used when stacking times - stack_by_datasets : bool (or None) - Flag to specify what is plotted ("stacked") together per subfigure. - If True, stack datasets together, otherwise stack by times. If - None, stack_by_datasets will be set based on the number of times - and datasets. - labelsubplots : bool, list or tuple - Label subplots as (a), (b), (c), ... If a list or tuple is given - their values should be the horizontal and vertical position - relative to each subaxis. - showlegend : bool (or None) - Label different plots and show legend. If None, showlegend is set - to True if legend will have more than one entry, otherwise it is - set to False. - fieldorder : 'C' or 'F' - Index ordering for assigning fields and datasets/times (depending - on stack_by_datasets) to axes grid (row by row). Fields is considered the - first axis, so 'C' means fields change slowest, 'F' means fields - change fastest. - ncols : int - Number of columns in axes grid, must be a true divisor of total - number of axes. - subfigsize : list or tuple - Standard size of subfigures - plot_local_time : bool or str - Plot dual x axes with both UTC time and local time. If a str is - provided, then plot_local_time is assumed to be True and the str - is used as the datetime format. - local_time_offset : float - Local time offset from UTC - datasetkwargs : dict - Dataset-specific options that are passed on to the actual - plotting function. These options overwrite general options - specified through **kwargs. The argument should be a dictionary - with entries : {**kwargs} - **kwargs : other keyword arguments - Options that are passed on to the actual plotting function. - Note that these options should be the same for all datasets, - fields and times, and they can not be used to set dataset, - field or time specific colors, limits, etc. - Example uses include setting linestyle/width, marker, etc. - """ - - args = PlottingInput( - datasets=datasets, - fields=fields, - times=times, - timerange=timerange, - fieldlimits=fieldlimits, - fieldlabels=fieldlabels, - fieldorder=fieldorder, - ) - - nfields = len(args.fields) - ntimes = len(args.times) - ndatasets = len(args.datasets) - - # Concatenate custom and standard field labels - # (custom field labels overwrite standard fields labels if existent) - args.fieldlabels = {**standard_fieldlabels, **args.fieldlabels} - - # Set up subplot grid - if stack_by_datasets is None: - if ntimes>1: - stack_by_datasets = False - else: - stack_by_datasets = True - - if stack_by_datasets: - ntotal = nfields * ntimes - else: - ntotal = nfields * ndatasets - - fig, ax, nrows, ncols = _create_subplots_if_needed( - ntotal, - ncols, - default_ncols=int(ntotal/nfields), - fieldorder=args.fieldorder, - avoid_single_column=True, - sharey=True, - subfigsize=subfigsize, - hspace=0.4, - fig=fig, - ax=ax, - ) - - # Create flattened view of axes - axv = np.asarray(ax).reshape(-1) - - # Set showlegend if not specified - if showlegend is None: - if (stack_by_datasets and ndatasets>1) or (not stack_by_datasets and ntimes>1): - showlegend = True - else: - showlegend = False - - # Set default sequential colormap if timerange was specified - if (timerange is not None) and (cmap is None): - cmap = 'viridis' - - # Loop over datasets, fields and times - for i, dfname in enumerate(args.datasets): - df = args.datasets[dfname] - heightvalues = _get_dim_values(df,'height',default_idx=True) - assert(heightvalues is not None), 'profile plot needs a height axis' - timevalues = _get_dim_values(df,'time') - - # If plot local time, shift timevalues - timedelta_to_local = None - if plot_local_time is not False: - timedelta_to_local = pd.to_timedelta(local_time_offset,'h') - timevalues = timevalues + timedelta_to_local - - # Create list with available fields only - available_fields = _get_available_fieldnames(df,args.fields) - - # Pivot all fields in a dataset at once - if timevalues is not None: - df_pivot = _get_pivot_table(df,'height',available_fields) - - for j, field in enumerate(args.fields): - # If available_fields is [None,], fieldname is unimportant - if available_fields == [None]: - pass - # Else, check if field is available - elif not field in available_fields: - print('Warning: field "'+field+'" not available in dataset '+dfname) - continue - - for k, time in enumerate(args.times): - plotting_properties = {} - - # Axis order, label and title depend on value of stack_by_datasets - if stack_by_datasets: - # Index of axis corresponding to field j and time k - if args.fieldorder == 'C': - axi = j*ntimes + k - else: - axi = k*nfields + j - - # Use datasetname as label - if showlegend: - plotting_properties['label'] = dfname - - # Set title if multiple times are compared - if ntimes>1: - if isinstance(time, (int,float,np.number)): - tstr = '{:g} s'.format(time) - else: - if plot_local_time is False: - tstr = pd.to_datetime(time).strftime('%Y-%m-%d %H%M UTC') - elif plot_local_time is True: - tstr = pd.to_datetime(time).strftime('%Y-%m-%d %H:%M') - else: - assert isinstance(plot_local_time,str), 'Unexpected plot_local_time format' - tstr = pd.to_datetime(time).strftime(plot_local_time) - axv[axi].set_title(tstr, fontsize=16) - - # Set color - plotting_properties['color'] = default_colors[i % len(default_colors)] - else: - # Index of axis corresponding to field j and dataset i - if args.fieldorder == 'C': - axi = j*ndatasets + i - else: - axi = i*nfields + j - - # Use time as label - if showlegend: - if isinstance(time, (int,float,np.number)): - plotting_properties['label'] = '{:g} s'.format(time) - else: - if plot_local_time is False: - plotting_properties['label'] = pd.to_datetime(time).strftime('%Y-%m-%d %H%M UTC') - elif plot_local_time is True: - plotting_properties['label'] = pd.to_datetime(time).strftime('%Y-%m-%d %H:%M') - else: - assert isinstance(plot_local_time,str), 'Unexpected plot_local_time format' - plotting_properties['label'] = pd.to_datetime(time).strftime(plot_local_time) - - # Set title if multiple datasets are compared - if ndatasets>1: - axv[axi].set_title(dfname,fontsize=16) - - # Set colors - if cmap is not None: - cmap = mpl.cm.get_cmap(cmap) - plotting_properties['color'] = cmap(k/(ntimes-1)) - else: - plotting_properties['color'] = default_colors[k % len(default_colors)] - - # Extract data from dataframe - if timevalues is None: - # Dataset will not be pivoted - fieldvalues = _get_field(df,field).values - else: - if plot_local_time is not False: - # specified times are in local time, convert back to UTC - slice_t = _get_slice(df_pivot,time-timedelta_to_local,'time') - else: - slice_t = _get_slice(df_pivot,time,'time') - fieldvalues = _get_pivoted_field(slice_t,field).values.squeeze() - - # Gather label, color, general options and dataset-specific options - # (highest priority to dataset-specific options, then general options) - try: - plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} - except KeyError: - plotting_properties = {**plotting_properties,**kwargs} - - # Plot data - try: - axv[axi].plot(fieldvalues,heightvalues,**plotting_properties) - except ValueError as e: - print(e,'--', time, 'not found in index?') - - # Set field label if known - try: - axv[axi].set_xlabel(args.fieldlabels[field]) - except KeyError: - pass - # Set field limits if specified - try: - axv[axi].set_xlim(args.fieldlimits[field]) - except KeyError: - pass - - for axi in axv: - axi.grid(True,which='both') - - # Set height limits if specified - if not heightlimits is None: - axv[0].set_ylim(heightlimits) - - # Add y labels - for r in range(nrows): - axv[r*ncols].set_ylabel(r'Height [m]') - - # Align labels - _align_labels(fig,axv,nrows,ncols) - - # Number sub figures as a, b, c, ... - if labelsubplots is not False: - try: - hoffset, voffset = labelsubplots - except (TypeError, ValueError): - hoffset, voffset = -0.14, -0.18 - for i,axi in enumerate(axv): - axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) - - # Add legend - if showlegend: - leg = _format_legend(axv,index=ncols-1) - - return fig,ax - - -def plot_spectrum(datasets, - fields=None, - height=None, - times=None, - fig=None,ax=None, - fieldlimits=None, - freqlimits=None, - fieldlabels={}, - labelsubplots=False, - showlegend=None, - ncols=None, - subfigsize=(4,5), - datasetkwargs={}, - **kwargs - ): - """ - Plot frequency spectrum at a given height for different datasets, - time(s) and field(s), using a subplot per time and per field. - - Note that this function does not interpolate to the requested height, - i.e., if height is not None, the specified value should be available - in all datasets. - - Usage - ===== - datasets : pandas.DataFrame or dict - Dataset(s) with spectrum data. If more than one set, - datasets should be a dictionary with entries - : dataset - fields : str, list, 'all' (or None) - Fieldname(s) corresponding to particular column(s) of - the datasets. fields can be None if input are Series. - 'all' means all fields will be plotted (in this case all - datasets should have the same fields) - height : float (or None) - Height for which frequency spectra is plotted. If datasets - have no height dimension, height does not need to be specified. - times : str, int, float, list (or None) - Time(s) for which frequency spectra are plotted, specified as - either datetime strings or numerical values (seconds, e.g., - simulation time). times can be None if all datasets combined - have no more than one time value. - fig : figure handle - Custom figure handle. Should be specified together with ax - ax : axes handle, or list or numpy ndarray with axes handles - Customand axes handle(s). - Size of ax should equal nfields * ntimes - fieldlimits : list or tuple, or dict - Value range for the various fields. If only one field is - plotted, fieldlimits can be a list or tuple. Otherwise, it - should be a dictionary with entries : fieldlimit. - Missing fieldlimits are set automatically - freqlimits : list or tuple - Frequency axis limits - fieldlabels : str or dict - Custom field labels. If only one field is plotted, fieldlabels - can be a string. Otherwise it should be a dictionary with - entries : fieldlabel - labelsubplots : bool, list or tuple - Label subplots as (a), (b), (c), ... If a list or tuple is given - their values should be the horizontal and vertical position - relative to each subaxis. - showlegend : bool (or None) - Label different plots and show legend. If None, showlegend is set - to True if legend will have more than one entry, otherwise it is - set to False. - ncols : int - Number of columns in axes grid, must be a true divisor of total - number of axes. - subfigsize : list or tuple - Standard size of subfigures - datasetkwargs : dict - Dataset-specific options that are passed on to the actual - plotting function. These options overwrite general options - specified through **kwargs. The argument should be a dictionary - with entries : {**kwargs} - **kwargs : other keyword arguments - Options that are passed on to the actual plotting function. - Note that these options should be the same for all datasets, - fields and times, and they can not be used to set dataset, - field or time specific colors, limits, etc. - Example uses include setting linestyle/width, marker, etc. - """ - - args = PlottingInput( - datasets=datasets, - fields=fields, - times=times, - fieldlimits=fieldlimits, - fieldlabels=fieldlabels, - ) - - nfields = len(args.fields) - ntimes = len(args.times) - ndatasets = len(args.datasets) - ntotal = nfields * ntimes - - # Concatenate custom and standard field labels - # (custom field labels overwrite standard fields labels if existent) - args.fieldlabels = {**standard_spectrumlabels, **args.fieldlabels} - - fig, ax, nrows, ncols = _create_subplots_if_needed( - ntotal, - ncols, - default_ncols=ntimes, - avoid_single_column=True, - sharex=True, - subfigsize=subfigsize, - wspace=0.3, - fig=fig, - ax=ax, - ) - - # Create flattened view of axes - axv = np.asarray(ax).reshape(-1) - - # Set showlegend if not specified - if showlegend is None: - if ndatasets>1: - showlegend = True - else: - showlegend = False - - # Loop over datasets, fields and times - for i, dfname in enumerate(args.datasets): - df = args.datasets[dfname] - - frequencyvalues = _get_dim_values(df,'frequency',default_idx=True) - assert(frequencyvalues is not None), 'spectrum plot needs a frequency axis' - timevalues = _get_dim_values(df,'time') - - # Create list with available fields only - available_fields = _get_available_fieldnames(df,args.fields) - - for j, field in enumerate(args.fields): - # If available_fields is [None,], fieldname is unimportant - if available_fields == [None]: - pass - # Else, check if field is available - elif not field in available_fields: - print('Warning: field "'+field+'" not available in dataset '+dfname) - continue - - for k, time in enumerate(args.times): - plotting_properties = {} - if showlegend: - plotting_properties['label'] = dfname - - # Index of axis corresponding to field j and time k - axi = j*ntimes + k - - # Axes mark up - if i==0 and ntimes>1: - axv[axi].set_title(pd.to_datetime(time).strftime('%Y-%m-%d %H%M UTC'),fontsize=16) - - # Gather label, general options and dataset-specific options - # (highest priority to dataset-specific options, then general options) - try: - plotting_properties = {**plotting_properties,**kwargs,**datasetkwargs[dfname]} - except KeyError: - plotting_properties = {**plotting_properties,**kwargs} - - # Get field spectrum - slice_t = _get_slice(df,time,'time') - slice_tz = _get_slice(slice_t,height,'height') - spectrum = _get_field(slice_tz,field).values - - # Plot data - axv[axi].loglog(frequencyvalues[1:],spectrum[1:],**plotting_properties) - - # Specify field limits if specified - try: - axv[axi].set_ylim(args.fieldlimits[field]) - except KeyError: - pass - - - # Set frequency label - for c in range(ncols): - axv[ncols*(nrows-1)+c].set_xlabel('$f$ [Hz]') - - # Specify field label if specified - for r in range(nrows): - try: - axv[r*ncols].set_ylabel(args.fieldlabels[args.fields[r]]) - except KeyError: - pass - - # Align labels - _align_labels(fig,axv,nrows,ncols) - - # Set frequency limits if specified - if not freqlimits is None: - axv[0].set_xlim(freqlimits) - - # Number sub figures as a, b, c, ... - if labelsubplots is not False: - try: - hoffset, voffset = labelsubplots - except (TypeError, ValueError): - hoffset, voffset = -0.14, -0.18 - for i,axi in enumerate(axv): - axi.text(hoffset,voffset,'('+chr(i+97)+')',transform=axi.transAxes,size=16) - - # Add legend - if showlegend: - leg = _format_legend(axv,index=ncols-1) - - return fig, ax - - -# --------------------------------------------- -# -# DEFINITION OF AUXILIARY CLASSES AND FUNCTIONS -# -# --------------------------------------------- - -class InputError(Exception): - """Exception raised for errors in the input. - - Attributes: - message -- explanation of the error - """ - - def __init__(self, message): - self.message = message - - -class PlottingInput(object): - """ - Auxiliary class to collect input data and options for plotting - functions, and to check if the inputs are consistent - """ - supported_datatypes = ( - pd.Series, - pd.DataFrame, - xr.DataArray, - xr.Dataset, - ) - - def __init__(self, datasets, fields, **argd): - # Add all arguments as class attributes - self.__dict__.update({'datasets':datasets, - 'fields':fields, - **argd}) - - # Check consistency of all attributes - self._check_consistency() - - def _check_consistency(self): - """ - Check consistency of all input data - """ - - # ---------------------- - # Check dataset argument - # ---------------------- - # If a single dataset is provided, convert to a dictionary - # under a generic key 'Dataset' - if isinstance(self.datasets, self.supported_datatypes): - self.datasets = {'Dataset': self.datasets} - for dfname,df in self.datasets.items(): - # convert dataset types here - if isinstance(df, (xr.Dataset,xr.DataArray)): - # handle xarray datatypes - self.datasets[dfname] = df.to_dataframe() - columns = self.datasets[dfname].columns - if len(columns) == 1: - # convert to pd.Series - self.datasets[dfname] = self.datasets[dfname][columns[0]] - else: - assert(isinstance(df, self.supported_datatypes)), \ - "Dataset {:s} of type {:s} not supported".format(dfname,str(type(df))) - - # ---------------------- - # Check fields argument - # ---------------------- - # If no fields are specified, check that - # - all datasets are series - # - the name of every series is either None or matches other series names - if self.fields is None: - assert(all([isinstance(self.datasets[dfname],pd.Series) for dfname in self.datasets])), \ - "'fields' argument must be specified unless all datasets are pandas Series" - series_names = set() - for dfname in self.datasets: - series_names.add(self.datasets[dfname].name) - if len(series_names)==1: - self.fields = list(series_names) - else: - raise InputError('attempting to plot multiple series with different field names') - elif isinstance(self.fields,str): - # If fields='all', retrieve fields from dataset - if self.fields=='all': - self.fields = _get_fieldnames(list(self.datasets.values())[0]) - assert(all([_get_fieldnames(df)==self.fields for df in self.datasets.values()])), \ - "The option fields = 'all' only works when all datasets have the same fields" - # If fields is a single instance, convert to a list - else: - self.fields = [self.fields,] - - # ---------------------------------- - # Check match of fields and datasets - # ---------------------------------- - # Check if all datasets have at least one of the requested fields - for dfname in self.datasets: - df = self.datasets[dfname] - if isinstance(df,pd.DataFrame): - assert(any([field in df.columns for field in self.fields])), \ - 'DataFrame '+dfname+' does not contain any of the requested fields' - elif isinstance(df,pd.Series): - if df.name is None: - assert(len(self.fields)==1), \ - 'Series must have a name if more than one fields is specified' - else: - assert(df.name in self.fields), \ - 'Series '+dfname+' does not match any of the requested fields' - - # --------------------------------- - # Check heights argument (optional) - # --------------------------------- - try: - # If no heights are specified, check that all datasets combined have - # no more than one height value - if self.heights is None: - av_heights = set() - for df in self.datasets.values(): - heightvalues = _get_dim_values(df,'height') - try: - for height in heightvalues: - av_heights.add(height) - except TypeError: - # heightvalues is None - pass - if len(av_heights)==0: - # None of the datasets have height values - self.heights = [None,] - elif len(av_heights)==1: - self.heights = list(av_heights) - else: - raise InputError("found more than one height value so 'heights' argument must be specified") - # If heights='all', retrieve heights from dataset - elif isinstance(self.heights,str) and self.heights=='all': - self.heights = _get_dim_values(list(self.datasets.values())[0],'height') - assert(all([np.allclose(_get_dim_values(df,'height'),self.heights) for df in self.datasets.values()])), \ - "The option heights = 'all' only works when all datasets have the same vertical levels" - # If heights is single instance, convert to list - elif isinstance(self.heights,(int,float)): - self.heights = [self.heights,] - except AttributeError: - pass - - # ----------------------------------- - # Check timerange argument (optional) - # ----------------------------------- - try: - if self.timerange is not None: - if self.times is not None: - print('Using specified time range',self.timerange, - 'and ignoring',self.times) - assert isinstance(self.timerange,(tuple,list)), \ - 'Need to specify timerange as (starttime,endtime)' - assert (len(self.timerange) == 2) - try: - starttime = pd.to_datetime(self.timerange[0]) - endtime = pd.to_datetime(self.timerange[1]) - except ValueError: - print('Unable to convert timerange to timestamps') - else: - # get unique times from all datasets - alltimes = [] - for df in self.datasets.values(): - alltimes += list(_get_dim_values(df,'time')) - alltimes = pd.DatetimeIndex(np.unique(alltimes)) - inrange = (alltimes >= starttime) & (alltimes <= endtime) - self.times = alltimes[inrange] - except AttributeError: - pass - - # --------------------------------- - # Check times argument (optional) - # --------------------------------- - # If times is single instance, convert to list - try: - # If no times are specified, check that all datasets combined have - # no more than one time value - if self.times is None: - av_times = set() - for df in self.datasets.values(): - timevalues = _get_dim_values(df,'time') - try: - for time in timevalues.values: - av_times.add(time) - except AttributeError: - pass - if len(av_times)==0: - # None of the datasets have time values - self.times = [None,] - elif len(av_times)==1: - self.times = list(av_times) - else: - raise InputError("found more than one time value so 'times' argument must be specified") - elif isinstance(self.times,(str,int,float,np.number,pd.Timestamp)): - self.times = [self.times,] - except AttributeError: - pass - - # ------------------------------------- - # Check fieldlimits argument (optional) - # ------------------------------------- - # If one set of fieldlimits is specified, check number of fields - # and convert to dictionary - try: - if self.fieldlimits is None: - self.fieldlimits = {} - elif isinstance(self.fieldlimits, (list, tuple)): - assert(len(self.fields)==1), 'Unclear to what field fieldlimits corresponds' - self.fieldlimits = {self.fields[0]:self.fieldlimits} - except AttributeError: - self.fieldlimits = {} - - # ------------------------------------- - # Check fieldlabels argument (optional) - # ------------------------------------- - # If one fieldlabel is specified, check number of fields - try: - if isinstance(self.fieldlabels, str): - assert(len(self.fields)==1), 'Unclear to what field fieldlabels corresponds' - self.fieldlabels = {self.fields[0]: self.fieldlabels} - except AttributeError: - self.fieldlabels = {} - - # ------------------------------------- - # Check colorscheme argument (optional) - # ------------------------------------- - # If one colorscheme is specified, check number of fields - try: - self.cmap = {} - if isinstance(self.colorschemes, str): - assert(len(self.fields)==1), 'Unclear to what field colorschemes corresponds' - self.cmap[self.fields[0]] = mpl.cm.get_cmap(self.colorschemes) - else: - # Set missing colorschemes to viridis - for field in self.fields: - if field not in self.colorschemes.keys(): - if field == 'wdir': - self.colorschemes[field] = 'twilight' - else: - self.colorschemes[field] = 'viridis' - self.cmap[field] = mpl.cm.get_cmap(self.colorschemes[field]) - except AttributeError: - pass - - # ------------------------------------- - # Check fieldorder argument (optional) - # ------------------------------------- - # Make sure fieldorder is recognized - try: - assert(self.fieldorder in ['C','F']), "Error: fieldorder '"\ - +self.fieldorder+"' not recognized, must be either 'C' or 'F'" - except AttributeError: - pass - - - def set_missing_fieldlimits(self): - """ - Set missing fieldlimits to min and max over all datasets - """ - for field in self.fields: - if field not in self.fieldlimits.keys(): - try: - self.fieldlimits[field] = [ - min([_get_field(df,field).min() for df in self.datasets.values() if _contains_field(df,field)]), - max([_get_field(df,field).max() for df in self.datasets.values() if _contains_field(df,field)]) - ] - except ValueError: - self.fieldlimits[field] = [None,None] - -def _get_dim(df,dim,default_idx=False): - """ - Search for specified dimension in dataset and return - level (referred to by either label or position) and - axis {0 or ‘index’, 1 or ‘columns’} - - If default_idx is True, return a single unnamed index - if present - """ - assert(dim in dimension_names.keys()), \ - "Dimension '"+dim+"' not supported" - - # 1. Try to find dim based on name - for name in dimension_names[dim]: - if name in df.index.names: - if debug: print("Found "+dim+" dimension in index with name '{}'".format(name)) - return name, 0 - else: - try: - if name in df.columns: - if debug: print("Found "+dim+" dimension in column with name '{}'".format(name)) - return name, 1 - except AttributeError: - # pandas Series has no columns - pass - - # 2. Look for Datetime or Timedelta index - if dim=='time': - for idx in range(len(df.index.names)): - if isinstance(df.index.get_level_values(idx),(pd.DatetimeIndex,pd.TimedeltaIndex,pd.PeriodIndex)): - if debug: print("Found "+dim+" dimension in index with level {} without a name ".format(idx)) - return idx, 0 - - # 3. If default index is True, assume that a - # single nameless index corresponds to the - # requested dimension - if (not isinstance(df.index,(pd.MultiIndex,pd.DatetimeIndex,pd.TimedeltaIndex,pd.PeriodIndex)) - and default_idx and (df.index.name is None) ): - if debug: print("Assuming nameless index corresponds to '{}' dimension".format(dim)) - return 0,0 - - # 4. Did not found requested dimension - if debug: print("Found no "+dim+" dimension") - return None, None - - -def _get_available_fieldnames(df,fieldnames): - """ - Return subset of fields available in df - """ - available_fieldnames = [] - if isinstance(df,pd.DataFrame): - for field in fieldnames: - if field in df.columns: - available_fieldnames.append(field) - # A Series only has one field, so return that field name - # (if that field is not in fields, an error would have been raised) - elif isinstance(df,pd.Series): - available_fieldnames.append(df.name) - return available_fieldnames - - -def _get_fieldnames(df): - """ - Return list of fieldnames in df - """ - if isinstance(df,pd.DataFrame): - fieldnames = list(df.columns) - # Remove any column corresponding to - # a dimension (time, height or frequency) - for dim in dimension_names.keys(): - name, axis = _get_dim(df,dim) - if axis==1: - fieldnames.remove(name) - return fieldnames - elif isinstance(df,pd.Series): - return [df.name,] - - -def _contains_field(df,fieldname): - if isinstance(df,pd.DataFrame): - return fieldname in df.columns - elif isinstance(df,pd.Series): - return (df.name is None) or (df.name==fieldname) - - -def _get_dim_values(df,dim,default_idx=False): - """ - Return values for a given dimension - """ - level, axis = _get_dim(df,dim,default_idx) - # Requested dimension is an index - if axis==0: - return df.index.get_level_values(level).unique() - # Requested dimension is a column - elif axis==1: - return df[level].unique() - # Requested dimension not available - else: - return None - - -def _get_pivot_table(df,dim,fieldnames): - """ - Return pivot table with given fieldnames as columns - """ - level, axis = _get_dim(df,dim) - # Unstack an index - if axis==0: - return df.unstack(level=level) - # Pivot about a column - elif axis==1: - return df.pivot(columns=level,values=fieldnames) - # Dimension not found, return dataframe - else: - return df - - -def _get_slice(df,key,dim): - """ - Return cross-section of dataset - """ - if key is None: - return df - - # Get dimension level and axis - level, axis = _get_dim(df,dim) - - # Requested dimension is an index - if axis==0: - if isinstance(df.index,pd.MultiIndex): - return df.xs(key,level=level) - else: - return df.loc[df.index==key] - # Requested dimension is a column - elif axis==1: - return df.loc[df[level]==key] - # Requested dimension not available, return dataframe - else: - return df - - -def _get_field(df,fieldname): - """ - Return field from dataset - """ - if isinstance(df,pd.DataFrame): - return df[fieldname] - elif isinstance(df,pd.Series): - if df.name is None or df.name==fieldname: - return df - else: - return None - - -def _get_pivoted_field(df,fieldname): - """ - Return field from pivoted dataset - """ - if isinstance(df.columns,pd.MultiIndex): - return df[fieldname] - else: - return df - - -def _create_subplots_if_needed(ntotal, - ncols=None, - default_ncols=1, - fieldorder='C', - avoid_single_column=False, - sharex=False, - sharey=False, - subfigsize=(12,3), - wspace=0.2, - hspace=0.2, - fig=None, - ax=None - ): - """ - Auxiliary function to create fig and ax - - If fig and ax are None: - - Set nrows and ncols based on ntotal and specified ncols, - accounting for fieldorder and avoid_single_column - - Create fig and ax with nrows and ncols, taking into account - sharex, sharey, subfigsize, wspace, hspace - - If fig and ax are not None: - - Try to determine nrows and ncols from ax - - Check whether size of ax corresponds to ntotal - """ - - if ax is None: - if not ncols is None: - # Use ncols if specified and appropriate - assert(ntotal%ncols==0), 'Error: Specified number of columns is not a true divisor of total number of subplots' - nrows = int(ntotal/ncols) - else: - # Defaut number of columns - ncols = default_ncols - nrows = int(ntotal/ncols) - - if fieldorder=='F': - # Swap number of rows and columns - nrows, ncols = ncols, nrows - - if avoid_single_column and ncols==1: - # Swap number of rows and columns - nrows, ncols = ncols, nrows - - # Create fig and ax with nrows and ncols - fig,ax = plt.subplots(nrows=nrows,ncols=ncols,sharex=sharex,sharey=sharey,figsize=(subfigsize[0]*ncols,subfigsize[1]*nrows)) - - # Adjust subplot spacing - fig.subplots_adjust(wspace=wspace,hspace=hspace) - - else: - # Make sure user-specified axes has appropriate size - assert(np.asarray(ax).size==ntotal), 'Specified axes does not have the right size' - - # Determine nrows and ncols in specified axes - if isinstance(ax,mpl.axes.Axes): - nrows, ncols = (1,1) - else: - try: - nrows,ncols = np.asarray(ax).shape - except ValueError: - # ax array has only one dimension - # Determine whether ax is single row or single column based - # on individual ax positions x0 and y0 - x0s = [axi.get_position().x0 for axi in ax] - y0s = [axi.get_position().y0 for axi in ax] - if all(x0==x0s[0] for x0 in x0s): - # All axis have same relative x0 position - nrows = np.asarray(ax).size - ncols = 1 - elif all(y0==y0s[0] for y0 in y0s): - # All axis have same relative y0 position - nrows = 1 - ncols = np.asarray(ax).size - else: - # More complex axes configuration, - # currently not supported - raise InputError('could not determine nrows and ncols in specified axes, complex axes configuration currently not supported') - - return fig, ax, nrows, ncols - - -def _format_legend(axv,index): - """ - Auxiliary function to format legend - - Usage - ===== - axv : numpy 1d array - Flattened array of axes - index : int - Index of the axis where to place the legend - """ - all_handles = [] - all_labels = [] - # Check each axes and add new handle - for axi in axv: - handles, labels = axi.get_legend_handles_labels() - for handle,label in zip(handles,labels): - if not label in all_labels: - all_labels.append(label) - all_handles.append(handle) - - leg = axv[index].legend(all_handles,all_labels,loc='upper left',bbox_to_anchor=(1.05,1.0),fontsize=16) - return leg - - -def _format_time_axis(fig,ax, - plot_local_time, - local_time_offset, - timelimits - ): - """ - Auxiliary function to format time axis - """ - ax[-1].xaxis_date() - if timelimits is not None: - timelimits = [pd.to_datetime(tlim) for tlim in timelimits] - hour_interval = _determine_hourlocator_interval(ax[-1],timelimits) - if plot_local_time is not False: - if plot_local_time is True: - localtimefmt = '%I %p' - else: - assert isinstance(plot_local_time,str), 'Unexpected plot_local_time format' - localtimefmt = plot_local_time - # Format first axis (local time) - ax[-1].xaxis.set_minor_locator(mdates.HourLocator(byhour=range(0,24,hour_interval))) - ax[-1].xaxis.set_minor_formatter(mdates.DateFormatter(localtimefmt)) - ax[-1].xaxis.set_major_locator(mdates.DayLocator(interval=12)) #Choose large interval so dates are not plotted - ax[-1].xaxis.set_major_formatter(mdates.DateFormatter('')) - - # Set time limits if specified - if not timelimits is None: - local_timelimits = pd.to_datetime(timelimits) + pd.to_timedelta(local_time_offset,'h') - ax[-1].set_xlim(local_timelimits) - - tstr = 'Local time' - - ax2 = [] - for axi in ax: - # Format second axis (UTC time) - ax2i = axi.twiny() - ax2i.xaxis_date() - - # Set time limits if specified - if not timelimits is None: - ax2i.set_xlim(timelimits) - else: - # Extract timelimits from main axis - local_timelimits = mdates.num2date(axi.get_xlim()) - timelimits = pd.to_datetime(local_timelimits) - pd.to_timedelta(local_time_offset,'h') - ax2i.set_xlim(timelimits) - - # Move twinned axis ticks and label from top to bottom - ax2i.xaxis.set_ticks_position("bottom") - ax2i.xaxis.set_label_position("bottom") - - # Offset the twin axis below the host - ax2i.spines["bottom"].set_position(("axes", -0.35)) - - # Turn on the frame for the twin axis, but then hide all - # but the bottom spine - ax2i.set_frame_on(True) - ax2i.patch.set_visible(False) - #for sp in ax2.spines.itervalues(): - # sp.set_visible(False) - ax2i.spines["bottom"].set_visible(True) - - ax2i.xaxis.set_minor_locator(mdates.HourLocator(byhour=range(24),interval=hour_interval)) - ax2i.xaxis.set_minor_formatter(mdates.DateFormatter('%H%M')) - ax2i.xaxis.set_major_locator(mdates.DayLocator()) - ax2i.xaxis.set_major_formatter(mdates.DateFormatter('\n%Y-%m-%d')) - ax2i.set_xlabel('UTC time') - - ax2.append(ax2i) - - if len(ax2)==1: - ax2 = ax2[0] - else: - ax2 = np.array(ax2) - fig.align_xlabels(ax2) - else: - ax[-1].xaxis.set_minor_locator(mdates.HourLocator(byhour=range(0,24,hour_interval))) - ax[-1].xaxis.set_minor_formatter(mdates.DateFormatter('%H%M')) - ax[-1].xaxis.set_major_locator(mdates.DayLocator()) - ax[-1].xaxis.set_major_formatter(mdates.DateFormatter('\n%Y-%m-%d')) - - # Set time limits if specified - if not timelimits is None: - ax[-1].set_xlim(timelimits) - - tstr = 'UTC time' - ax2 = None - - # Now, update all axes - for axi in ax: - # Make sure both major and minor axis labels are visible when they are - # at the same time - axi.xaxis.remove_overlapping_locs = False - - # Set time label - axi.set_xlabel(tstr) - - return ax2 - - -def _determine_hourlocator_interval(ax,timelimits=None): - """ - Determine hour interval based on timelimits - - If plotted time period is - - less than 36 hours: interval = 3 - - less than 72 hours: interval = 6 - - otherwise: interval = 12 - """ - # Get timelimits - if timelimits is None: - timelimits = pd.to_datetime(mdates.num2date(ax.get_xlim())) - elif isinstance(timelimits[0],str): - timelimits = pd.to_datetime(timelimits) - - # Determine time period in hours - timeperiod = (timelimits[1] - timelimits[0])/pd.to_timedelta(1,'h') - # HourLocator interval - if timeperiod < 36: - return 3 - elif timeperiod < 72: - return 6 - else: - return 12 - - -def _get_staggered_grid(x): - """ - Return staggered grid locations - - For input array size N, output array - has a size of N+1 - """ - idx = np.arange(x.size) - f = interp1d(idx,x,fill_value='extrapolate') - return f(np.arange(-0.5,x.size+0.5,1)) - - -def _align_labels(fig,ax,nrows,ncols): - """ - Align labels of a given axes grid - """ - # Align xlabels row by row - for r in range(nrows): - fig.align_xlabels(ax[r*ncols:(r+1)*ncols]) - # Align ylabels column by column - for c in range(ncols): - fig.align_ylabels(ax[c::ncols]) - - -class TaylorDiagram(object): - """ - Taylor diagram. - - Plot model standard deviation and correlation to reference (data) - sample in a single-quadrant polar plot, with r=stddev and - theta=arccos(correlation). - - Based on code from Yannick Copin - Downloaded from https://gist.github.com/ycopin/3342888 on 2020-06-19 - """ - - def __init__(self, refstd, - fig=None, rect=111, label='_', srange=(0, 1.5), extend=False, - normalize=False, - corrticks=[0, 0.2, 0.4, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1], - minorcorrticks=None, - stdevticks=None, - labelsize=None): - """ - Set up Taylor diagram axes, i.e. single quadrant polar - plot, using `mpl_toolkits.axisartist.floating_axes`. - - Usage - ===== - refstd: np.ndarray - Reference standard deviation to be compared to - fig: plt.Figure, optional - Input figure or None to create a new figure - rect: 3-digit integer - Subplot position, described by: nrows, ncols, index - label: str, optional - Legend label for reference point - srange: tuple, optional - Stdev axis limits, in units of *refstd* - extend: bool, optional - Extend diagram to negative correlations - normalize: bool, optional - Normalize stdev axis by `refstd` - corrticks: list-like, optional - Specify ticks positions on azimuthal correlation axis - minorcorrticks: list-like, optional - Specify minor tick positions on azimuthal correlation axis - stdevticks: int or list-like, optional - Specify stdev axis grid locator based on MaxNLocator (with - integer input) or FixedLocator (with list-like input) - labelsize: int or str, optional - Font size (e.g., 16 or 'x-large') for all axes labels - """ - - from matplotlib.projections import PolarAxes - from mpl_toolkits.axisartist import floating_axes - from mpl_toolkits.axisartist import grid_finder - - self.refstd = refstd # Reference standard deviation - self.normalize = normalize - - tr = PolarAxes.PolarTransform() - - # Correlation labels - if minorcorrticks is None: - rlocs = np.array(corrticks) - else: - rlocs = np.array(sorted(list(corrticks) + list(minorcorrticks))) - if extend: - # Diagram extended to negative correlations - self.tmax = np.pi - rlocs = np.concatenate((-rlocs[:0:-1], rlocs)) - else: - # Diagram limited to positive correlations - self.tmax = np.pi/2 - if minorcorrticks is None: - rlocstrs = [str(rloc) for rloc in rlocs] - else: - rlocstrs = [str(rloc) if abs(rloc) in corrticks else '' - for rloc in rlocs] - tlocs = np.arccos(rlocs) # Conversion to polar angles - gl1 = grid_finder.FixedLocator(tlocs) # Positions - tf1 = grid_finder.DictFormatter(dict(zip(tlocs, rlocstrs))) - - # Stdev labels - if isinstance(stdevticks, int): - gl2 = grid_finder.MaxNLocator(stdevticks) - elif hasattr(stdevticks, '__iter__'): - gl2 = grid_finder.FixedLocator(stdevticks) - else: - gl2 = None - - # Standard deviation axis extent (in units of reference stddev) - self.smin, self.smax = srange - if not normalize: - self.smin *= self.refstd - self.smax *= self.refstd - - ghelper = floating_axes.GridHelperCurveLinear( - tr, - extremes=(0, self.tmax, self.smin, self.smax), - grid_locator1=gl1, - grid_locator2=gl2, - tick_formatter1=tf1, - #tick_formatter2=tf2 - ) - - if fig is None: - fig = plt.figure() - - ax = floating_axes.FloatingSubplot(fig, rect, grid_helper=ghelper) - fig.add_subplot(ax) - - # Adjust axes - # - angle axis - ax.axis["top"].set_axis_direction("bottom") - ax.axis["top"].toggle(ticklabels=True, label=True) - ax.axis["top"].major_ticklabels.set_axis_direction("top") - ax.axis["top"].label.set_axis_direction("top") - ax.axis["top"].label.set_text("Correlation") - - # - "x" axis - ax.axis["left"].set_axis_direction("bottom") - if normalize: - ax.axis["left"].label.set_text("Normalized standard deviation") - else: - ax.axis["left"].label.set_text("Standard deviation") - - # - "y" axis - ax.axis["right"].set_axis_direction("top") # "Y-axis" - ax.axis["right"].toggle(ticklabels=True) - ax.axis["right"].major_ticklabels.set_axis_direction( - "bottom" if extend else "left") - - # Set label sizes - if labelsize is not None: - ax.axis["top"].label.set_fontsize(labelsize) - ax.axis["left"].label.set_fontsize(labelsize) - ax.axis["right"].label.set_fontsize(labelsize) - ax.axis["top"].major_ticklabels.set_fontsize(labelsize) - ax.axis["left"].major_ticklabels.set_fontsize(labelsize) - ax.axis["right"].major_ticklabels.set_fontsize(labelsize) - - if self.smin: - # get rid of cluster of labels at origin - ax.axis["bottom"].toggle(ticklabels=False, label=False) - else: - ax.axis["bottom"].set_visible(False) # Unused - - self._ax = ax # Graphical axes - self.ax = ax.get_aux_axes(tr) # Polar coordinates - - # Add reference point and stddev contour - t = np.linspace(0, self.tmax) - r = np.ones_like(t) - if self.normalize: - l, = self.ax.plot([0], [1], 'k*', ls='', ms=10, label=label) - else: - l, = self.ax.plot([0], self.refstd, 'k*', ls='', ms=10, label=label) - r *= refstd - self.ax.plot(t, r, 'k--', label='_') - - # Collect sample points for latter use (e.g. legend) - self.samplePoints = [l] - - def set_ref(self, refstd): - """ - Update the reference standard deviation value - - Useful for cases in which datasets with different reference - values (e.g., originating from different reference heights) - are to be overlaid on the same diagram. - """ - self.refstd = refstd - - def add_sample(self, stddev, corrcoef, norm=None, *args, **kwargs): - """ - Add sample (*stddev*, *corrcoeff*) to the Taylor - diagram. *args* and *kwargs* are directly propagated to the - `Figure.plot` command. - - `norm` may be specified to override the default normalization - value if TaylorDiagram was initialized with normalize=True - """ - if (corrcoef < 0) and (self.tmax == np.pi/2): - print('Note: ({:g},{:g}) not shown for R2 < 0, set extend=True'.format(stddev,corrcoef)) - return None - - if self.normalize: - if norm is None: - norm = self.refstd - elif norm is False: - norm = 1 - stddev /= norm - - l, = self.ax.plot(np.arccos(corrcoef), stddev, - *args, **kwargs) # (theta, radius) - self.samplePoints.append(l) - - return l - - def add_grid(self, *args, **kwargs): - """Add a grid.""" - - self._ax.grid(*args, **kwargs) - - def add_contours(self, levels=5, scale=1.0, **kwargs): - """ - Add constant centered RMS difference contours, defined by *levels*. - """ - - rs, ts = np.meshgrid(np.linspace(self.smin, self.smax), - np.linspace(0, self.tmax)) - # Compute centered RMS difference - if self.normalize: - # - normalized refstd == 1 - # - rs values were previously normalized in __init__ - # - premultiply with (scale==refstd) to get correct rms diff - rms = scale * np.sqrt(1 + rs**2 - 2*rs*np.cos(ts)) - else: - rms = np.sqrt(self.refstd**2 + rs**2 - 2*self.refstd*rs*np.cos(ts)) - - contours = self.ax.contour(ts, rs, rms, levels, **kwargs) - - return contours - - def set_xlabel(self, label, fontsize=None): - """ - Set the label for the standard deviation axis - """ - self._ax.axis["left"].label.set_text(label) - if fontsize is not None: - self._ax.axis["left"].label.set_fontsize(fontsize) - - def set_alabel(self, label, fontsize=None): - """ - Set the label for the azimuthal axis - """ - self._ax.axis["top"].label.set_text(label) - if fontsize is not None: - self._ax.axis["top"].label.set_fontsize(fontsize) - - def set_title(self, label, **kwargs): - """ - Set the title for the axes - """ - self._ax.set_title(label, **kwargs) - diff --git a/mmctools/plotting/__init__.py b/mmctools/plotting/__init__.py new file mode 100644 index 0000000..7f5669d --- /dev/null +++ b/mmctools/plotting/__init__.py @@ -0,0 +1,2 @@ +# enable from mmctools.plotting import ... for backwards compatibility +from ..windtools.windtools.plotting import * From cc1180432e5684799931013eb0a4bdbdfafeb6b5 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Thu, 29 Apr 2021 17:46:07 -0600 Subject: [PATCH 071/145] Slightly less hacky way of providing windtools submodules Use cases: ``` from mmctools.plotting import plot_timeheight # from mmctools/windtools/windtools/plotting.py from mmctools.SOWFA6.postProcessing.probeSets import ProbeSets # from mmctools/windtools/windtools/SOWFA6/postProcessing/probeSets.py ``` --- mmctools/__init__.py | 24 ++++++++++++++++++++++++ mmctools/plotting/__init__.py | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 mmctools/__init__.py delete mode 100644 mmctools/plotting/__init__.py diff --git a/mmctools/__init__.py b/mmctools/__init__.py new file mode 100644 index 0000000..37e1830 --- /dev/null +++ b/mmctools/__init__.py @@ -0,0 +1,24 @@ + +#import windtools.windtools.plotting as plotting +# +# +# -- the wrong module was imported! + +#from .windtools.windtools import plotting +#print(plotting) +# +# +# ImportError: cannot import name 'plot_*' from 'mmctools.plotting' (unknown location) + +#from .windtools.windtools import * +#print(plotting) +# +# NameError: name 'plotting' is not defined + + +# for backwards compatibility, enable `from mmctools.plotting import plot_something` +# enable `from mmctools.foo.bar import baz # to import baz from windtools.foo.bar` +import os +mmctools_module = os.path.split(__file__)[0] +__path__.append(os.path.join(mmctools_module,'windtools','windtools')) + diff --git a/mmctools/plotting/__init__.py b/mmctools/plotting/__init__.py deleted file mode 100644 index 7f5669d..0000000 --- a/mmctools/plotting/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# enable from mmctools.plotting import ... for backwards compatibility -from ..windtools.windtools.plotting import * From 897373486de2da1caa4600e83f0a66edc58f3bba Mon Sep 17 00:00:00 2001 From: William Lassman Date: Fri, 28 May 2021 09:11:48 -0700 Subject: [PATCH 072/145] add wrfout_slices_seriesReader function --- mmctools/wrf/utils.py | 147 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index e351ac8..89ff87b 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1304,6 +1304,153 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter, return ds_subset + +def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, + specified_height = None, + do_slice_vars = True, + do_surf_vars = False, + vlist = None ): + """ + Construct an a2e-mmc standard, xarrays-based, data structure from a + series of WRF slice outpput fies + + Note: Base state theta= 300.0 K is assumed by convention in WRF, + this function follow this convention. + + Usage + ==== + wrfpath : string + The path to directory containing wrfout files to be processed + wrf_file_filter : string-glob expression + A string-glob expression to filter a set of 4-dimensional WRF + output files. + specified_height : list-like, optional + If not None, then a list of static heights to which all data + variables should be interpolated. Note that this significantly + increases the data read time. + do_slice_vars: Logical (default True), optional + If true, then the slice variables (SLICES_U, SLICES_V, SLICES_W, SLICES_T) are read for the specified height + (or for all heights if 'specified_height = None') + do_surf_vars: Logical (default False), optional + If true, then the surface variables (UST, HFX, QFX, SST, SSTK) will be added to the file + vlist: List-like, default None (optional). + If not none, then do_slice_vars and do_surf_vars set to False, and only variables in the list 'vlist' are read + """ + TH0 = 300.0 #WRF convention base-state theta = 300.0 K + dims_dict = { + 'Time':'datetime', + 'num_slices':'nz_slice', + 'south_north': 'ny', + 'west_east':'nx', + } + + ds = xr.open_mfdataset(os.path.join(wrf_path,wrf_file_filter), + chunks={'Time': 10}, + combine='nested', + concat_dim='Time') + + ds = ds.assign_coords({"SLICES_Z": ds.SLICES_Z.isel(Time=1)}) + ds = ds.swap_dims({'num_slices': 'SLICES_Z'}) + + dim_keys = ["Time","bottom_top","south_north","west_east"] + horiz_dim_keys = ["south_north","west_east"] + print('Finished opening/concatenating datasets...') + #print(ds.dims) + ds_subset = ds[['Time']] + print('Establishing coordinate variables, x,y,z, zSurface...') + ycoord = ds.DY * (0.5 + np.arange(ds.dims['south_north'])) + xcoord = ds.DX * (0.5 + np.arange(ds.dims['west_east'])) + ds_subset['z'] = xr.DataArray(specified_height, dims='num_slices') + + ds_subset['y'] = xr.DataArray(ycoord, dims='south_north') + ds_subset['x'] = xr.DataArray(xcoord, dims='west_east') + + + + if vlist not None: + print("Vlist not nne, setting do_slice_vars andd do_surf_vars to False") + print("Does not support specified_height argument, grabing all available heights") + do_slice_vars = False + do_surf_vars = False + print("Extracting variables") + for vv in vlist: + print(vv) + ds_subset[vv] = ds[vv] + + + if do_slice_vars: + print("Doing slice variables") + print('Grabbing u, v, w, T') + if specified_height is not None: + if len(specified_height) == 1: + print("One height") + #print(ds.dims) + #print(ds.coords) + ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_height ) + + ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_height ) + + ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_height ) + + ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_height ) + else: + print("Multiple heights") + ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_height ) + + ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_height ) + + ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_height ) + + ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_height ) + else: + + ds_subset['u'] = ds['SLICES_U'] + + ds_subset['v'] = ds['SLICES_V'] + + ds_subset['w'] = ds['SLICES_W'] + + ds_subset['T'] = ds['SLICES_T'] + print('Calculating derived data variables, wspd, wdir...') + #print( (ds_subset['u'].ufuncs.square()).values ) + ds_subset['wspd'] = xr.DataArray(np.sqrt(ds_subset['u'].values**2 + ds_subset['v'].values**2), + dims=dim_keys) + ds_subset['wdir'] = xr.DataArray(180. + np.arctan2(ds_subset['u'].values,ds_subset['v'].values)*180./np.pi, + dims=dim_keys) + + + + + if do_surf_vars: + print('Extracting 2-D variables (UST, HFX, QFX, SST, SSTSK)') + ds_subset['UST'] = ds['UST'] + ds_subset['HFX'] = ds['HFX'] + ds_subset['QFX'] = ds['QFX'] + ds_subset['SST'] = ds['SST'] + ds_subset['SSTK'] = ds['SSTK'] + else: + print("Skipping 2-D variables") + + + + # assign rename coord variable for time, and assign ccordinates + + if specified_height is None: + ds_subset = ds_subset.assign_coords(z=ds_subset['SLICES_Z']) + ds_subset = ds_subset.assign_coords(y=ds_subset['y']) + ds_subset = ds_subset.assign_coords(x=ds_subset['x']) + + + + print(ds_subset.dims) + ds_subset = ds_subset.rename_dims(dims_dict) + + return ds_subset + + + + + def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): """ Write a list of lat/lon or i/j locations to a tslist file that is From 6c8254300bd4ebf9906a9db09fc5186e56702fca Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 12:30:08 -0600 Subject: [PATCH 073/145] Rename specified_height to specified_heights for consistency with existing wrfout_seriesReader() function --- mmctools/wrf/utils.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 89ff87b..dbbcac4 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1306,7 +1306,7 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter, def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, - specified_height = None, + specified_heights = None, do_slice_vars = True, do_surf_vars = False, vlist = None ): @@ -1324,13 +1324,13 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, wrf_file_filter : string-glob expression A string-glob expression to filter a set of 4-dimensional WRF output files. - specified_height : list-like, optional + specified_heights : list-like, optional If not None, then a list of static heights to which all data variables should be interpolated. Note that this significantly increases the data read time. do_slice_vars: Logical (default True), optional If true, then the slice variables (SLICES_U, SLICES_V, SLICES_W, SLICES_T) are read for the specified height - (or for all heights if 'specified_height = None') + (or for all heights if 'specified_heights = None') do_surf_vars: Logical (default False), optional If true, then the surface variables (UST, HFX, QFX, SST, SSTK) will be added to the file vlist: List-like, default None (optional). @@ -1360,7 +1360,7 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, print('Establishing coordinate variables, x,y,z, zSurface...') ycoord = ds.DY * (0.5 + np.arange(ds.dims['south_north'])) xcoord = ds.DX * (0.5 + np.arange(ds.dims['west_east'])) - ds_subset['z'] = xr.DataArray(specified_height, dims='num_slices') + ds_subset['z'] = xr.DataArray(specified_heights, dims='num_slices') ds_subset['y'] = xr.DataArray(ycoord, dims='south_north') ds_subset['x'] = xr.DataArray(xcoord, dims='west_east') @@ -1369,7 +1369,7 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, if vlist not None: print("Vlist not nne, setting do_slice_vars andd do_surf_vars to False") - print("Does not support specified_height argument, grabing all available heights") + print("Does not support specified_heights argument, grabing all available heights") do_slice_vars = False do_surf_vars = False print("Extracting variables") @@ -1381,27 +1381,27 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, if do_slice_vars: print("Doing slice variables") print('Grabbing u, v, w, T') - if specified_height is not None: - if len(specified_height) == 1: + if specified_heights is not None: + if len(specified_heights) == 1: print("One height") #print(ds.dims) #print(ds.coords) - ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_height ) + ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_heights ) - ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_height ) + ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_heights ) - ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_height ) + ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_heights ) - ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_height ) + ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_heights ) else: print("Multiple heights") - ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_height ) + ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_heights ) - ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_height ) + ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_heights ) - ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_height ) + ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_heights ) - ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_height ) + ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_heights ) else: ds_subset['u'] = ds['SLICES_U'] @@ -1435,7 +1435,7 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, # assign rename coord variable for time, and assign ccordinates - if specified_height is None: + if specified_heights is None: ds_subset = ds_subset.assign_coords(z=ds_subset['SLICES_Z']) ds_subset = ds_subset.assign_coords(y=ds_subset['y']) ds_subset = ds_subset.assign_coords(x=ds_subset['x']) From 6b326c9689d509698ddeca2439d33d82eafcfc4e Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 12:50:22 -0600 Subject: [PATCH 074/145] Cleanup docstring --- mmctools/wrf/utils.py | 61 ++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index dbbcac4..89d1c9d 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1304,37 +1304,38 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter, return ds_subset - def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, - specified_heights = None, - do_slice_vars = True, - do_surf_vars = False, - vlist = None ): - """ - Construct an a2e-mmc standard, xarrays-based, data structure from a - series of WRF slice outpput fies - - Note: Base state theta= 300.0 K is assumed by convention in WRF, - this function follow this convention. - - Usage - ==== - wrfpath : string - The path to directory containing wrfout files to be processed - wrf_file_filter : string-glob expression - A string-glob expression to filter a set of 4-dimensional WRF - output files. - specified_heights : list-like, optional - If not None, then a list of static heights to which all data - variables should be interpolated. Note that this significantly - increases the data read time. - do_slice_vars: Logical (default True), optional - If true, then the slice variables (SLICES_U, SLICES_V, SLICES_W, SLICES_T) are read for the specified height - (or for all heights if 'specified_heights = None') - do_surf_vars: Logical (default False), optional - If true, then the surface variables (UST, HFX, QFX, SST, SSTK) will be added to the file - vlist: List-like, default None (optional). - If not none, then do_slice_vars and do_surf_vars set to False, and only variables in the list 'vlist' are read + specified_heights=None, + do_slice_vars=True, + do_surf_vars=False, + vlist=None): + """ + Construct an a2e-mmc standard, xarrays-based, data structure from a + series of WRF slice output files + + Note: Base state theta= 300.0 K is assumed by convention in WRF, + and this function follows this convention. + Usage + ==== + wrfpath : string + The path to directory containing wrfout files to be processed + wrf_file_filter : string-glob expression + A string-glob expression to filter a set of 4-dimensional WRF + output files. + specified_heights : list-like, optional + If not None, then a list of static heights to which all data + variables should be interpolated. Note that this significantly + increases the data read time. + do_slice_vars: Logical (default True), optional + If true, then the slice variables (SLICES_U, SLICES_V, SLICES_W, + SLICES_T) are read for the specified height (or for all heights + if 'specified_heights = None') + do_surf_vars: Logical (default False), optional + If true, then the surface variables (UST, HFX, QFX, SST, SSTK) + will be added to the file + vlist: List-like, default None (optional) + If not none, then do_slice_vars and do_surf_vars set to False, + and only variables in the list 'vlist' are read """ TH0 = 300.0 #WRF convention base-state theta = 300.0 K dims_dict = { From c6dc326c92aef96e270c3e11421c3aa1de249516 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 12:52:05 -0600 Subject: [PATCH 075/145] Clarify vlist option behavior --- mmctools/wrf/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 89d1c9d..27cb841 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1334,7 +1334,7 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, If true, then the surface variables (UST, HFX, QFX, SST, SSTK) will be added to the file vlist: List-like, default None (optional) - If not none, then do_slice_vars and do_surf_vars set to False, + If not none, then set do_slice_vars and do_surf_vars to False, and only variables in the list 'vlist' are read """ TH0 = 300.0 #WRF convention base-state theta = 300.0 K @@ -1366,10 +1366,8 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, ds_subset['y'] = xr.DataArray(ycoord, dims='south_north') ds_subset['x'] = xr.DataArray(xcoord, dims='west_east') - - if vlist not None: - print("Vlist not nne, setting do_slice_vars andd do_surf_vars to False") + print("vlist not None, setting do_slice_vars and do_surf_vars to False") print("Does not support specified_heights argument, grabing all available heights") do_slice_vars = False do_surf_vars = False @@ -1378,7 +1376,6 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, print(vv) ds_subset[vv] = ds[vv] - if do_slice_vars: print("Doing slice variables") print('Grabbing u, v, w, T') From bbcc27f8efde278607a620299927a1209220cd74 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 12:54:25 -0600 Subject: [PATCH 076/145] Clean up formatting/style --- mmctools/wrf/utils.py | 57 +++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 27cb841..40544a3 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1382,42 +1382,32 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, if specified_heights is not None: if len(specified_heights) == 1: print("One height") - #print(ds.dims) - #print(ds.coords) - ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_heights ) - - ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_heights ) - - ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_heights ) - - ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_heights ) + #print(ds.dims) + #print(ds.coords) + ds_subset['u'] = ds['SLICES_U'].sel(SLICES_Z=specified_heights) + ds_subset['v'] = ds['SLICES_V'].sel(SLICES_Z=specified_heights) + ds_subset['w'] = ds['SLICES_W'].sel(SLICES_Z=specified_heights) + ds_subset['T'] = ds['SLICES_T'].sel(SLICES_Z=specified_heights) else: print("Multiple heights") - ds_subset['u'] = ds['SLICES_U'].sel( SLICES_Z = specified_heights ) - - ds_subset['v'] = ds['SLICES_V'].sel( SLICES_Z = specified_heights ) - - ds_subset['w'] = ds['SLICES_W'].sel( SLICES_Z = specified_heights ) - - ds_subset['T'] = ds['SLICES_T'].sel( SLICES_Z = specified_heights ) + ds_subset['u'] = ds['SLICES_U'].sel(SLICES_Z=specified_heights) + ds_subset['v'] = ds['SLICES_V'].sel(SLICES_Z=specified_heights) + ds_subset['w'] = ds['SLICES_W'].sel(SLICES_Z=specified_heights) + ds_subset['T'] = ds['SLICES_T'].sel(SLICES_Z=specified_heights) else: - ds_subset['u'] = ds['SLICES_U'] - ds_subset['v'] = ds['SLICES_V'] - ds_subset['w'] = ds['SLICES_W'] - ds_subset['T'] = ds['SLICES_T'] - print('Calculating derived data variables, wspd, wdir...') - #print( (ds_subset['u'].ufuncs.square()).values ) - ds_subset['wspd'] = xr.DataArray(np.sqrt(ds_subset['u'].values**2 + ds_subset['v'].values**2), - dims=dim_keys) - ds_subset['wdir'] = xr.DataArray(180. + np.arctan2(ds_subset['u'].values,ds_subset['v'].values)*180./np.pi, - dims=dim_keys) - - + print('Calculating derived data variables, wspd, wdir...') + #print((ds_subset['u'].ufuncs.square()).values) + ds_subset['wspd'] = xr.DataArray( + np.sqrt(ds_subset['u'].values**2 + ds_subset['v'].values**2), + dims=dim_keys) + ds_subset['wdir'] = xr.DataArray( + 180. + np.arctan2(ds_subset['u'].values,ds_subset['v'].values)*180./np.pi, + dims=dim_keys) if do_surf_vars: print('Extracting 2-D variables (UST, HFX, QFX, SST, SSTSK)') @@ -1429,26 +1419,17 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, else: print("Skipping 2-D variables") - - - # assign rename coord variable for time, and assign ccordinates - + # assign rename coord variable for time, and assign coordinates if specified_heights is None: ds_subset = ds_subset.assign_coords(z=ds_subset['SLICES_Z']) ds_subset = ds_subset.assign_coords(y=ds_subset['y']) ds_subset = ds_subset.assign_coords(x=ds_subset['x']) - - - print(ds_subset.dims) ds_subset = ds_subset.rename_dims(dims_dict) return ds_subset - - - def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): """ Write a list of lat/lon or i/j locations to a tslist file that is From 3b290bd34f99ce11460a1ed0fc0234ad8ba8f030 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 13:00:40 -0600 Subject: [PATCH 077/145] Fix syntax and inconsistent tabs --- mmctools/wrf/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 40544a3..60c672a 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1366,10 +1366,10 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, ds_subset['y'] = xr.DataArray(ycoord, dims='south_north') ds_subset['x'] = xr.DataArray(xcoord, dims='west_east') - if vlist not None: - print("vlist not None, setting do_slice_vars and do_surf_vars to False") - print("Does not support specified_heights argument, grabing all available heights") - do_slice_vars = False + if vlist is not None: + print("vlist not None, setting do_slice_vars and do_surf_vars to False") + print("Does not support specified_heights argument, grabing all available heights") + do_slice_vars = False do_surf_vars = False print("Extracting variables") for vv in vlist: @@ -1430,6 +1430,10 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, return ds_subset +def test(blerg): + TH0 = 300.0 #WRF convention base-state theta = 300.0 K + print(TH0) + def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): """ Write a list of lat/lon or i/j locations to a tslist file that is From b7a0167ce666bc7012f8999b4bce3444e104c0ad Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 13:03:44 -0600 Subject: [PATCH 078/145] Define module-level TH0 = 300.0 --- mmctools/wrf/utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 60c672a..df90502 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -60,6 +60,8 @@ 'UST', # u* from M-O ] +TH0 = 300.0 # [K] base-state potential temperature by WRF convention + def _get_dim(wrfdata,dimname): """Returns the specified dimension, with support for both netCDF4 and xarray @@ -370,7 +372,7 @@ def _create_datadict(self,varns,unstagger=False,staggered_vars=['ph']): datadict[varn] = ((tsdata[:,1:] + tsdata[:,:-1]) / 2).ravel() elif varn == 'th': # theta is a special case - #assert np.all(tsdata[:,-1] == 300), 'Unexpected nonzero value for theta' + #assert np.all(tsdata[:,-1] == TH0), 'Unexpected nonzero value for theta' # drop the trailing 0 for already unstaggered quantities datadict[varn] = tsdata[:,:-1].ravel() else: @@ -564,7 +566,7 @@ def interp_to_heights(df): tsdata = getattr(self,varn) #if varn == 'th': # # theta is a special case - # assert np.all(tsdata[:,-1] == 300) + # assert np.all(tsdata[:,-1] == TH0) #elif not varn == 'ww': # # if w has already been destaggered by wrf # assert np.all(tsdata[:,-1] == 0) @@ -744,7 +746,7 @@ def add_surface_plane(var,plane=None): def extract_column_from_wrfdata(fpath, coords, Ztop=2000., Vres=5.0, - T0=300., + T0=TH0, spatial_filter='interpolate',L_filter=0.0, additional_fields=[], verbose=False, @@ -1189,7 +1191,6 @@ def wrfout_seriesReader(wrf_path,wrf_file_filter, dimension to facilitate and expedite xarray operations """ import wrf as wrfpy - TH0 = 300.0 #WRF convention base-state theta = 300.0 K dims_dict = { 'Time':'datetime', 'bottom_top':'nz', @@ -1337,7 +1338,6 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, If not none, then set do_slice_vars and do_surf_vars to False, and only variables in the list 'vlist' are read """ - TH0 = 300.0 #WRF convention base-state theta = 300.0 K dims_dict = { 'Time':'datetime', 'num_slices':'nz_slice', @@ -1431,7 +1431,6 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, def test(blerg): - TH0 = 300.0 #WRF convention base-state theta = 300.0 K print(TH0) def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): From 834766a587ab51ba2ef7b0f470db6625502260f5 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 28 May 2021 13:09:15 -0600 Subject: [PATCH 079/145] Forgot to remove test function --- mmctools/wrf/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index df90502..9bc69cc 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1430,9 +1430,6 @@ def wrfout_slices_seriesReader(wrf_path, wrf_file_filter, return ds_subset -def test(blerg): - print(TH0) - def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): """ Write a list of lat/lon or i/j locations to a tslist file that is From 6db7140df4d122642f557eaee4ccdc1516692c76 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Mon, 12 Jul 2021 23:28:06 -0600 Subject: [PATCH 080/145] Update error checking when initializing CDSDataset --- mmctools/wrf/preprocessing.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index ad7d63f..8e7fd20 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -250,10 +250,15 @@ class CDSDataset(object): def __init__(self): if not os.path.isfile(self.api_rc): - print('WARNING: '+self.api_rc+' not found') - print('Go to https://cds.climate.copernicus.eu/api-how-to for more information') - import cdsapi - self.client = cdsapi.Client() + raise FileNotFoundError(f"""Expected CDS API key in {self.api_rc} +Go to https://cds.climate.copernicus.eu/api-how-to for more information""") + try: + import cdsapi + except ImportError: + raise ModuleNotFoundError("""Need CDS API client +Run `conda install -c conda-forge cdsapi`""") + else: + self.client = cdsapi.Client() def download(self,datetimes,product,prefix=None, variables=[], From 46f9bd3e5f1c0d2be307e6812a0d18b9c7967a8f Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 14 Jul 2021 14:32:39 -0600 Subject: [PATCH 081/145] Add combine_request kwarg For retrieving data at many output times, the previous approach (which creates one request per output time) can be rather slow; this combines all output datetimes into a single request (and output file). It's up to the user to decide whether to request days, weeks, months, or years at a time. --- mmctools/wrf/preprocessing.py | 44 ++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 8e7fd20..6f8e38f 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -263,7 +263,8 @@ def __init__(self): def download(self,datetimes,product,prefix=None, variables=[], area=[], - pressure_levels=None): + pressure_levels=None, + combine_request=False): """Download data at specified datetimes. Usage @@ -283,6 +284,11 @@ def download(self,datetimes,product,prefix=None, North/west/south/east lat/long limits pressure_levels : list, optional List of pressure levels + combine_request : bool, optional + Aggregate requested dates into lists of years, months, days, + and hours--note that this may return additional time steps + because the request selects all permutations of + year/month/day/hour; should be False for WRF WPS """ if prefix is None: prefix = os.path.join('.',product) @@ -297,14 +303,23 @@ def download(self,datetimes,product,prefix=None, if pressure_levels is not None: req['pressure_level'] = pressure_levels print('Requesting',len(pressure_levels),'pressure levels') - for datetime in datetimes: - req['year'] = datetime.strftime('%Y') - req['month'] = datetime.strftime('%m') - req['day'] = datetime.strftime('%d') - req['time'] = datetime.strftime('%H:%M') - target = datetime.strftime('{:s}_%Y_%m_%d_%H.grib'.format(prefix)) - #print(datetime,req,target) + if combine_request: + print('Combining all datetimes into a single request') + req['year'] = sorted(list(set([datetime.strftime('%Y') for datetime in datetimes]))) + req['month'] = sorted(list(set([datetime.strftime('%m') for datetime in datetimes]))) + req['day'] = sorted(list(set([datetime.strftime('%d') for datetime in datetimes]))) + req['time'] = sorted(list(set([datetime.strftime('%H:%M') for datetime in datetimes]))) + target = datetimes[0].strftime('{:s}_from_%Y_%m_%d_%H.grib'.format(prefix)) self.client.retrieve(product, req, target) + else: + for datetime in datetimes: + req['year'] = datetime.strftime('%Y') + req['month'] = datetime.strftime('%m') + req['day'] = datetime.strftime('%d') + req['time'] = datetime.strftime('%H:%M') + target = datetime.strftime('{:s}_%Y_%m_%d_%H.grib'.format(prefix)) + #print(datetime,req,target) + self.client.retrieve(product, req, target) class ERA5(CDSDataset): @@ -324,7 +339,7 @@ class ERA5(CDSDataset): Ref: https://confluence.ecmwf.int/pages/viewpage.action?pageId=74764925 """ - def download(self,datetimes,path=None,bounds={}): + def download(self,datetimes,path=None,bounds={},combine_request=False): """Download data at specified datetimes. Descriptions: @@ -344,6 +359,11 @@ def download(self,datetimes,path=None,bounds={}): includes all of US and Central America, most of Alaska and Canada (up to 60deg latitude), and parts of South America that lie north of the equator. + combine_request : bool, optional + Aggregate requested dates into lists of years, months, days, + and hours--note that this may return additional time steps + because the request selects all permutations of + year/month/day/hour; should be False for WRF WPS """ if path is None: path = '.' @@ -376,7 +396,8 @@ def download(self,datetimes,path=None,bounds={}): '600','650','700','750','775','800','825','850','875','900', '925','950','975','1000' ], - area=area + area=area, + combine_request=combine_request, ) super().download( datetimes, @@ -402,7 +423,8 @@ def download(self,datetimes,path=None,bounds={}): 'volumetric_soil_water_layer_1','volumetric_soil_water_layer_2', 'volumetric_soil_water_layer_3','volumetric_soil_water_layer_4' ], - area=area + area=area, + combine_request=combine_request, ) From 8fdb417c783c3079352e6c39f48467c5d31f8491 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Thu, 15 Jul 2021 10:34:48 -0600 Subject: [PATCH 082/145] Update install instructions --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d1fb88..ef9871c 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,18 @@ df = read_dir(dpath, file_filter='*_w*', reader=profiler) ## Installation -To install, run `pip install -e mmctools` after cloning the repository (or `pip install -e .` from inside a2e-mmc/mmctools). +The recommended approach is to first create a new conda environment: +``` +conda create -n mmc python=3.7 +conda activate mmc +conda install -y -c conda-forge jupyterlab matplotlib scipy xarray dask pyarrow gdal rasterio elevation pyyaml netcdf4 wrf-python cdsapi cfgrib +``` +Then create an "editable" installation of the mmctools repository: +``` +cd /path/to/a2e-mmc/mmctools +pip install -e .` +``` ## Code Development Principles From 8c1f661a9fc37d66dbef63e1d8baddc5e52c9718 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Thu, 15 Jul 2021 11:02:58 -0600 Subject: [PATCH 083/145] Add details about dependencies --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index ef9871c..81372d8 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,17 @@ conda create -n mmc python=3.7 conda activate mmc conda install -y -c conda-forge jupyterlab matplotlib scipy xarray dask pyarrow gdal rasterio elevation pyyaml netcdf4 wrf-python cdsapi cfgrib ``` +Note: All packages after `xarray` are optional: +- `dask` makes netcdf data processing more efficient +- `pyarrow` is a dependency for the "feather" data format, an *extremely* efficient + way to save dataframe data (in terms file I/O time and file size) +- `gdal`, `rasterio`, and `elevation` are required for processing terrain data +- `netcdf4` and `wrf-python` are for the NCAR-provided WRF utilities, which are + useful for interpolating and slicing data +- `cdsapi` is needed for `wrf.preprocessing` to retrieve Copernicus ERA5 + reanalysis data +- `cfgrib` enables xarray to load grib files + Then create an "editable" installation of the mmctools repository: ``` From 4e66c292d4c071d3f4771e8cdc6d0371f2eeb30a Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Thu, 15 Jul 2021 11:04:50 -0600 Subject: [PATCH 084/145] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 81372d8..e352225 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Note: All packages after `xarray` are optional: Then create an "editable" installation of the mmctools repository: ``` cd /path/to/a2e-mmc/mmctools -pip install -e .` +pip install -e . ``` ## Code Development Principles From 31978406ee5abeb77d9cdefe1944b2bab45c1842 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Thu, 15 Jul 2021 13:25:43 -0600 Subject: [PATCH 085/145] Add kwarg to select output slices and quantities of interest Added default variables (at pressure-levels or single/surface level) and default pressure levels --- mmctools/wrf/preprocessing.py | 128 +++++++++++++++++++++------------- 1 file changed, 78 insertions(+), 50 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 6f8e38f..caf1c61 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -339,7 +339,49 @@ class ERA5(CDSDataset): Ref: https://confluence.ecmwf.int/pages/viewpage.action?pageId=74764925 """ - def download(self,datetimes,path=None,bounds={},combine_request=False): + default_single_level_vars = [ + '10m_u_component_of_wind','10m_v_component_of_wind', + '2m_dewpoint_temperature','2m_temperature', + 'convective_snowfall','convective_snowfall_rate_water_equivalent', + 'ice_temperature_layer_1','ice_temperature_layer_2', + 'ice_temperature_layer_3','ice_temperature_layer_4', + 'land_sea_mask','large_scale_snowfall', + 'large_scale_snowfall_rate_water_equivalent', + 'maximum_2m_temperature_since_previous_post_processing', + 'mean_sea_level_pressure', + 'minimum_2m_temperature_since_previous_post_processing', + 'sea_ice_cover','sea_surface_temperature','skin_temperature', + 'snow_albedo','snow_density','snow_depth','snow_evaporation', + 'snowfall','snowmelt','soil_temperature_level_1', + 'soil_temperature_level_2','soil_temperature_level_3', + 'soil_temperature_level_4','soil_type','surface_pressure', + 'temperature_of_snow_layer','total_column_snow_water', + 'volumetric_soil_water_layer_1','volumetric_soil_water_layer_2', + 'volumetric_soil_water_layer_3','volumetric_soil_water_layer_4' + ] + + default_pressure_level_vars = [ + 'divergence','fraction_of_cloud_cover','geopotential', + 'ozone_mass_mixing_ratio','potential_vorticity', + 'relative_humidity','specific_cloud_ice_water_content', + 'specific_cloud_liquid_water_content','specific_humidity', + 'specific_rain_water_content','specific_snow_water_content', + 'temperature','u_component_of_wind','v_component_of_wind', + 'vertical_velocity','vorticity' + ] + + default_pressure_levels = [ + '1','2','3','5','7','10','20','30','50','70','100','125','150', + '175','200','225','250','300','350','400','450','500','550', + '600','650','700','750','775','800','825','850','875','900', + '925','950','975','1000' + ] + + def download(self,datetimes,path=None, + pressure_level_vars='default', pressure_levels='default', + single_level_vars='default', + bounds={}, + combine_request=False): """Download data at specified datetimes. Descriptions: @@ -359,6 +401,15 @@ def download(self,datetimes,path=None,bounds={},combine_request=False): includes all of US and Central America, most of Alaska and Canada (up to 60deg latitude), and parts of South America that lie north of the equator. + pressure_level_vars : list, optional + Variables to retrieve at the specified pressure levels; if + set to 'default', then use `default_pressure_level_vars` + pressure_levels : list, optional + Pressure levels from which 4D data are constructed; if set + to 'default', then use `default_pressure_levels` + single_level_vars : list, optional + Variables to retrieve at the specified pressure levels; if + set to 'default', then use `default_single_level_vars` combine_request : bool, optional Aggregate requested dates into lists of years, months, days, and hours--note that this may return additional time steps @@ -374,58 +425,35 @@ def download(self,datetimes,path=None,bounds={},combine_request=False): S_bound = bounds.get('S', 0) W_bound = bounds.get('W', -169) E_bound = bounds.get('E', -47) + + if single_level_vars == 'default': + single_level_vars = self.default_single_level_vars + if pressure_level_vars == 'default': + pressure_level_vars = self.default_pressure_level_vars + if pressure_levels == 'default': + pressure_levels = self.default_pressure_levels area = [N_bound, W_bound, S_bound, E_bound] - super().download( - datetimes, - 'reanalysis-era5-pressure-levels', - prefix=os.path.join(path,'era5_pressure'), - variables=[ - 'divergence','fraction_of_cloud_cover','geopotential', - 'ozone_mass_mixing_ratio','potential_vorticity', - 'relative_humidity','specific_cloud_ice_water_content', - 'specific_cloud_liquid_water_content','specific_humidity', - 'specific_rain_water_content','specific_snow_water_content', - 'temperature','u_component_of_wind','v_component_of_wind', - 'vertical_velocity','vorticity' - ], - pressure_levels=[ - '1','2','3','5','7','10','20','30','50','70','100','125','150', - '175','200','225','250','300','350','400','450','500','550', - '600','650','700','750','775','800','825','850','875','900', - '925','950','975','1000' - ], - area=area, - combine_request=combine_request, - ) - super().download( - datetimes, - 'reanalysis-era5-single-levels', - prefix=os.path.join(path,'era5_surface'), - variables=[ - '10m_u_component_of_wind','10m_v_component_of_wind', - '2m_dewpoint_temperature','2m_temperature', - 'convective_snowfall','convective_snowfall_rate_water_equivalent', - 'ice_temperature_layer_1','ice_temperature_layer_2', - 'ice_temperature_layer_3','ice_temperature_layer_4', - 'land_sea_mask','large_scale_snowfall', - 'large_scale_snowfall_rate_water_equivalent', - 'maximum_2m_temperature_since_previous_post_processing', - 'mean_sea_level_pressure', - 'minimum_2m_temperature_since_previous_post_processing', - 'sea_ice_cover','sea_surface_temperature','skin_temperature', - 'snow_albedo','snow_density','snow_depth','snow_evaporation', - 'snowfall','snowmelt','soil_temperature_level_1', - 'soil_temperature_level_2','soil_temperature_level_3', - 'soil_temperature_level_4','soil_type','surface_pressure', - 'temperature_of_snow_layer','total_column_snow_water', - 'volumetric_soil_water_layer_1','volumetric_soil_water_layer_2', - 'volumetric_soil_water_layer_3','volumetric_soil_water_layer_4' - ], - area=area, - combine_request=combine_request, - ) + if pressure_level_vars: + super().download( + datetimes, + 'reanalysis-era5-pressure-levels', + prefix=os.path.join(path,'era5_pressure'), + variables=pressure_level_vars, + pressure_levels=pressure_levels, + area=area, + combine_request=combine_request, + ) + if single_level_vars: + super().download( + datetimes, + 'reanalysis-era5-single-levels', + prefix=os.path.join(path,'era5_surface'), + variables=single_level_vars, + area=area, + combine_request=combine_request, + ) class SetupWRF(): From a1435528e06f4dbb6713f67a747dd776b581b2f9 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 21 Jul 2021 11:33:21 -0600 Subject: [PATCH 086/145] Fix bug on slope and vector rugeddness vector The underlying resolution of the height array wasn't being taken into account, which resulted in wrong computation of slope, rise-run, and VRM. The `richdem` call for slope also had `riserun` which is incorrect for this case. --- mmctools/helper_functions.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 9b036f1..aaf084e 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1323,7 +1323,7 @@ def tri_filt(x): return tri -def calcVRM(hgt,window=None,footprint=None,slope_zscale=1.0,return_slope=False): +def calcVRM(hgt,res,window=None,footprint=None,fill_depressions=True,return_slope_aspect=False): ''' Vector Ruggedness Measure Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). @@ -1333,6 +1333,9 @@ def calcVRM(hgt,window=None,footprint=None,slope_zscale=1.0,return_slope=False): hgt : array Array of heights over which TRI will be calculated + res : int or float + Resolution of the underlying hgt array. Should be constant in x and y + Needed for proper slope calculation window : int Length of window in x and y direction. Must be odd. ''' @@ -1353,18 +1356,22 @@ def calcVRM(hgt,window=None,footprint=None,slope_zscale=1.0,return_slope=False): assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) ny,nx = np.shape(hgt) + # Determine scale based on resolution of hgt array + zscale = 1/res + # Get slope and aspect: hgt_rd = rd.rdarray(hgt, no_data=-9999) - rd.FillDepressions(hgt_rd, in_place=True) - slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun', zscale=slope_zscale) + if fill_depressions: + rd.FillDepressions(hgt_rd, in_place=True) + slope = rd.TerrainAttribute(hgt_rd, attrib='slope_degrees',zscale=zscale) aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') # Calculate vectors: vrm = np.zeros((ny,nx)) - rugz = np.cos(slope*np.pi/180.0) - rugdxy = np.sin(slope*np.pi/180.0) - rugx = rugdxy*np.cos(aspect*np.pi/180.0) - rugy = rugdxy*np.sin(aspect*np.pi/180.0) + rugz = np.cos(np.deg2rad(slope)) + rugdxy = np.sin(np.deg2rad(slope)) + rugx = rugdxy*np.cos(np.deg2rad(aspect)) + rugy = rugdxy*np.sin(np.deg2rad(aspect)) def vrm_filt(x): return(sum(x)**2) @@ -1384,7 +1391,8 @@ def vrm_filt(x): else: num_points = float(window**2) vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/num_points - if return_slope: - return vrm,slope + if return_slope_aspect: + return vrm,slope,aspect else: return vrm + From 3ccb1467fbf432e4034929cd0649ad7de38c7ad5 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Fri, 23 Jul 2021 09:48:29 -0600 Subject: [PATCH 087/145] Move TRI and VRM functions from helper to coupling.terrain --- mmctools/coupling/terrain.py | 116 +++++++++++++++++++++++++++++++++++ mmctools/helper_functions.py | 115 ---------------------------------- 2 files changed, 116 insertions(+), 115 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 4d93d1b..c3f9a65 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -506,3 +506,119 @@ def calc_slope(x,y,z): rise_run = np.sqrt(dz_dx**2 + dz_dy**2) slope[1:-1,1:-1] = np.degrees(np.arctan(rise_run)) return slope + + +def calcTRI(hgt,window=None,footprint=None): + ''' + Terrain Ruggedness Index + Riley, S. J., DeGloria, S. D., & Elliot, R. (1999). Index that + quantifies topographic heterogeneity. intermountain Journal + of sciences, 5(1-4), 23-27. + + hgt : array + Array of heights over which TRI will be calculated + window : int + Length of window in x and y direction. Must be odd. + ''' + from scipy.ndimage.filters import generic_filter + + # Window setup: + if footprint is not None: + assert window is None, 'Must specify either window or footprint' + window = np.shape(footprint)[0] + + assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' + Hwindow = int(np.floor(window/2)) + + # Type and dimension check: + if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): + hgt = hgt.data + assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) + + ny,nx = np.shape(hgt) + + def tri_filt(x): + middle_ind = int(len(x)/2) + return((sum((x - x[middle_ind])**2.0))**0.5) + + if footprint is None: + tri = generic_filter(hgt,tri_filt, size = (window,window)) + else: + tri = generic_filter(hgt,tri_filt, footprint=footprint) + + return tri + + +def calcVRM(hgt,res,window=None,footprint=None,fill_depressions=True,return_slope_aspect=False): + ''' + Vector Ruggedness Measure + Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). + Quantifying landscape ruggedness for animal habitat analysis: + a case study using bighorn sheep in the Mojave Desert. The + Journal of wildlife management, 71(5), 1419-1426. + + hgt : array + Array of heights over which TRI will be calculated + res : int or float + Resolution of the underlying hgt array. Should be constant in x and y + Needed for proper slope calculation + window : int + Length of window in x and y direction. Must be odd. + ''' + import richdem as rd + from scipy.ndimage.filters import generic_filter + + # Window setup: + if footprint is not None: + assert window is None, 'Must specify either window or footprint' + window = np.shape(footprint)[0] + + assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' + Hwndw = int(np.floor(window/2)) + + # Type and dimension check: + if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): + hgt = hgt.data + assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) + ny,nx = np.shape(hgt) + + # Determine scale based on resolution of hgt array + zscale = 1/res + + # Get slope and aspect: + hgt_rd = rd.rdarray(hgt, no_data=-9999) + if fill_depressions: + rd.FillDepressions(hgt_rd, in_place=True) + slope = rd.TerrainAttribute(hgt_rd, attrib='slope_degrees',zscale=zscale) + aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') + + # Calculate vectors: + vrm = np.zeros((ny,nx)) + rugz = np.cos(np.deg2rad(slope)) + rugdxy = np.sin(np.deg2rad(slope)) + rugx = rugdxy*np.cos(np.deg2rad(aspect)) + rugy = rugdxy*np.sin(np.deg2rad(aspect)) + + def vrm_filt(x): + return(sum(x)**2) + + if footprint is None: + vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) + vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) + vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) + else: + vrmX = generic_filter(rugx,vrm_filt, footprint=footprint) + vrmY = generic_filter(rugy,vrm_filt, footprint=footprint) + vrmZ = generic_filter(rugz,vrm_filt, footprint=footprint) + + + if footprint is not None: + num_points = len(footprint[footprint != 0.0]) + else: + num_points = float(window**2) + vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/num_points + if return_slope_aspect: + return vrm,slope,aspect + else: + return vrm + diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index aaf084e..f3c75f2 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1281,118 +1281,3 @@ def calc_spectra(data, psd_f = psd_level.combine_first(psd_f) return(psd_f) - -def calcTRI(hgt,window=None,footprint=None): - ''' - Terrain Ruggedness Index - Riley, S. J., DeGloria, S. D., & Elliot, R. (1999). Index that - quantifies topographic heterogeneity. intermountain Journal - of sciences, 5(1-4), 23-27. - - hgt : array - Array of heights over which TRI will be calculated - window : int - Length of window in x and y direction. Must be odd. - ''' - from scipy.ndimage.filters import generic_filter - - # Window setup: - if footprint is not None: - assert window is None, 'Must specify either window or footprint' - window = np.shape(footprint)[0] - - assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' - Hwindow = int(np.floor(window/2)) - - # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): - hgt = hgt.data - assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) - - ny,nx = np.shape(hgt) - - def tri_filt(x): - middle_ind = int(len(x)/2) - return((sum((x - x[middle_ind])**2.0))**0.5) - - if footprint is None: - tri = generic_filter(hgt,tri_filt, size = (window,window)) - else: - tri = generic_filter(hgt,tri_filt, footprint=footprint) - - return tri - - -def calcVRM(hgt,res,window=None,footprint=None,fill_depressions=True,return_slope_aspect=False): - ''' - Vector Ruggedness Measure - Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). - Quantifying landscape ruggedness for animal habitat analysis: - a case study using bighorn sheep in the Mojave Desert. The - Journal of wildlife management, 71(5), 1419-1426. - - hgt : array - Array of heights over which TRI will be calculated - res : int or float - Resolution of the underlying hgt array. Should be constant in x and y - Needed for proper slope calculation - window : int - Length of window in x and y direction. Must be odd. - ''' - import richdem as rd - from scipy.ndimage.filters import generic_filter - - # Window setup: - if footprint is not None: - assert window is None, 'Must specify either window or footprint' - window = np.shape(footprint)[0] - - assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' - Hwndw = int(np.floor(window/2)) - - # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): - hgt = hgt.data - assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) - ny,nx = np.shape(hgt) - - # Determine scale based on resolution of hgt array - zscale = 1/res - - # Get slope and aspect: - hgt_rd = rd.rdarray(hgt, no_data=-9999) - if fill_depressions: - rd.FillDepressions(hgt_rd, in_place=True) - slope = rd.TerrainAttribute(hgt_rd, attrib='slope_degrees',zscale=zscale) - aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') - - # Calculate vectors: - vrm = np.zeros((ny,nx)) - rugz = np.cos(np.deg2rad(slope)) - rugdxy = np.sin(np.deg2rad(slope)) - rugx = rugdxy*np.cos(np.deg2rad(aspect)) - rugy = rugdxy*np.sin(np.deg2rad(aspect)) - - def vrm_filt(x): - return(sum(x)**2) - - if footprint is None: - vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) - vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) - vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) - else: - vrmX = generic_filter(rugx,vrm_filt, footprint=footprint) - vrmY = generic_filter(rugy,vrm_filt, footprint=footprint) - vrmZ = generic_filter(rugz,vrm_filt, footprint=footprint) - - - if footprint is not None: - num_points = len(footprint[footprint != 0.0]) - else: - num_points = float(window**2) - vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/num_points - if return_slope_aspect: - return vrm,slope,aspect - else: - return vrm - From 378e80278dc79f4128b9c6895efc07d197700fe0 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Fri, 23 Jul 2021 10:09:22 -0600 Subject: [PATCH 088/145] Add xarray package for terrain functions --- mmctools/coupling/terrain.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index c3f9a65..899752c 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -520,6 +520,7 @@ def calcTRI(hgt,window=None,footprint=None): window : int Length of window in x and y direction. Must be odd. ''' + import xarray as xr from scipy.ndimage.filters import generic_filter # Window setup: @@ -566,6 +567,7 @@ def calcVRM(hgt,res,window=None,footprint=None,fill_depressions=True,return_slop Length of window in x and y direction. Must be odd. ''' import richdem as rd + import xarray as xr from scipy.ndimage.filters import generic_filter # Window setup: From 322dfb4f325aece3a6aa79b68206a327a40defe9 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 15:25:34 -0600 Subject: [PATCH 089/145] Add general Lidar object and Perdigao data reader --- mmctools/measurements/lidar.py | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 9b191f5..df3d526 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -5,4 +5,77 @@ """ import numpy as np import pandas as pd +import xarray as xr + +class LidarData(object): + def __init__(self,df,verbose=True): + """Lidar data described by range, azimuth, and elevation""" + self.verbose = verbose + self.df = df + self._check_coords() + + def _check_coords(self): + if all([coord in self.df.index.names + for coord in ['range','azimuth','elevation'] + ]): + if self.verbose: print('3D volumetric scan loaded') + elif 'range' not in self.df.index.names: + if self.verbose: print('Vertical scan loaded') + elif 'azimuth' not in self.df.index.names: + if self.verbose: print('RHI scan loaded') + elif 'elevation' not in self.df.index.names: + if self.verbose: print('PPI scan loaded') + else: + raise IndexError('Unexpected index levels in dataframe: '+str(self.df.index.names)) + + dr = self.df.index.levels[0][1] - self.df.index.levels[0][0] + if hasattr(self, 'range_gate_size'): + assert self.range_gate_size == dr + else: + self.range_gate_size = dr + self.rmax = self.df.index.levels[0][-1] + self.range_gate_size + + @property + def r(self): + return self.df.index.levels[0] + + @property + def az(self): + return self.df.index.levels[1] + + @property + def el(self): + return self.df.index.levels[2] + + +class Perdigao(LidarData): + """Galion scanning lidar""" + + def __init__(self, + fpath, + range_gate='Range gates', + azimuth='Azimuth angle', + elevation='Elevation angle', + range_gate_size=30., + **kwargs): + self.range_gate_size = range_gate_size + df = self._load(fpath,range_gate,azimuth,elevation) + super().__init__(df, **kwargs) + + def _load(self,fpath,range_gate,azimuth,elevation): + """Process a single scan in netcdf format""" + ds = xr.open_dataset(fpath) + assert (len(ds.dims)==1) and ('y' in ds.dims) + r = np.unique(ds[range_gate]) * self.range_gate_size + az = np.unique(ds[azimuth]) + el = np.unique(ds[elevation]) + df = ds.to_dataframe() + df = df.rename(columns={ + 'Range gates': 'range', + 'Azimuth angle': 'azimuth', + 'Elevation angle': 'elevation', + }) + df['range'] *= self.range_gate_size + df = df.set_index(['range','azimuth','elevation']).sort_index() + return df From dbaeb0244cf47969af62e74fb400899bb8e85079 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 15:27:48 -0600 Subject: [PATCH 090/145] Add getters to slice 3D lidar data --- mmctools/measurements/lidar.py | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index df3d526..445c8a2 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -47,6 +47,64 @@ def az(self): def el(self): return self.df.index.levels[2] + # slicers + def get(r=None, az=None, el=None): + if r is not None: + return get_range(r) + elif az is not None: + return get_azimuth(az) + elif el is not None: + return get_elevation(el) + + def get_range(self, r): + rs = self.df.index.levels[0] + if r < 0: + raise ValueError('Invalid range, r < 0') + elif r >= self.rmax: + raise ValueError(f'Invalid range, r >= {self.rmax}') + if r not in rs: + try: + idx = np.where(r < rs)[0][0] - 1 + except IndexError: + idx = len(rs) - 1 + r0 = self.df.index.levels[0][idx] + r1 = self.rmax + else: + r0 = self.df.index.levels[0][idx] + r1 = self.df.index.levels[0][idx+1] + assert (r >= r0) & (r < r1) + else: + idx = list(rs).index(r) + r0 = r + r1 = r + self.range_gate_size + if self.verbose: + print(f'getting range gate {idx} between {r0} and {r1}') + return self.df.xs(r0, level='range') + + def get_azimuth(self, az): + azs = self.df.index.levels[1] + if az < azs[0]: + raise ValueError(f'Invalid range, az < {azs[0]}') + elif az > azs[-1]: + raise ValueError(f'Invalid range, az > {azs[-1]}') + if az not in azs: + az = azs[np.argmin(np.abs(az - azs))] + if self.verbose: + print(f'getting nearest azimuth={az} deg') + return self.df.xs(az, level='azimuth') + + def get_elevation(self, el): + els = self.df.index.levels[2] + if el < els[0]: + raise ValueError(f'Invalid range, el < {els[0]}') + elif el > els[-1]: + raise ValueError(f'Invalid range, el > {els[-1]}') + if el not in els: + el = els[np.argmin(np.abs(el - els))] + if self.verbose: + print(f'getting nearest elevation={el} deg') + return self.df.xs(el, level='elevation') + class Perdigao(LidarData): """Galion scanning lidar""" From e28d73f469c169ef616a37e64dd83e914bb9d37f Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 15:30:18 -0600 Subject: [PATCH 091/145] Fix general lidar.get() --- mmctools/measurements/lidar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 445c8a2..d8d4705 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -48,13 +48,13 @@ def el(self): return self.df.index.levels[2] # slicers - def get(r=None, az=None, el=None): + def get(self, r=None, az=None, el=None): if r is not None: - return get_range(r) + return self.get_range(r) elif az is not None: - return get_azimuth(az) + return self.get_azimuth(az) elif el is not None: - return get_elevation(el) + return self.get_elevation(el) def get_range(self, r): rs = self.df.index.levels[0] From 3c4566d62c0ce2c86352cf95237b3813c43113b6 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 15:54:21 -0600 Subject: [PATCH 092/145] Calculate x,y,z from r,az,el --- mmctools/measurements/lidar.py | 46 +++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index d8d4705..cf85567 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -8,11 +8,15 @@ import xarray as xr class LidarData(object): - def __init__(self,df,verbose=True): + def __init__(self,df,range=None,azimuth=None,elevation=None,verbose=True): """Lidar data described by range, azimuth, and elevation""" self.verbose = verbose self.df = df + self.specified_range = range + self.specified_azimuth = azimuth + self.specified_elevation = elevation self._check_coords() + self._to_xyz() def _check_coords(self): if all([coord in self.df.index.names @@ -21,10 +25,20 @@ def _check_coords(self): if self.verbose: print('3D volumetric scan loaded') elif 'range' not in self.df.index.names: if self.verbose: print('Vertical scan loaded') + if self.specified_range is None: + print('`range` not specified, x/y/z will be invalid') + self.specified_range = 1 elif 'azimuth' not in self.df.index.names: if self.verbose: print('RHI scan loaded') + if self.specified_azimuth is None: + print('`azimuth` not specified, x/y/z may be invalid') + self.specified_azimuth = 0.0 elif 'elevation' not in self.df.index.names: if self.verbose: print('PPI scan loaded') + if self.specified_elevation is None: + if self.verbose: + print('`elevation` not specified, assuming el=0') + self.specified_elevation = 0.0 else: raise IndexError('Unexpected index levels in dataframe: '+str(self.df.index.names)) @@ -35,6 +49,26 @@ def _check_coords(self): self.range_gate_size = dr self.rmax = self.df.index.levels[0][-1] + self.range_gate_size + def _to_xyz(self): + try: + r = self.df.index.get_level_values('range') + except KeyError: + r = self.specified_range + try: + az = np.radians(270 - self.df.index.get_level_values('azimuth')) + except KeyError: + az = np.radians(self.specified_azimuth) + try: + el = np.radians(self.df.index.get_level_values('elevation')) + except KeyError: + el = np.radians(self.specified_elevation) + if not hasattr(self,'x'): + self.x = r * np.cos(az) * np.cos(el) + if not hasattr(self,'y'): + self.y = r * np.sin(az) * np.cos(el) + if not hasattr(self,'z'): + self.z = r * np.sin(el) + @property def r(self): return self.df.index.levels[0] @@ -111,13 +145,13 @@ class Perdigao(LidarData): def __init__(self, fpath, - range_gate='Range gates', - azimuth='Azimuth angle', - elevation='Elevation angle', + range_gate_name='Range gates', + azimuth_name='Azimuth angle', + elevation_name='Elevation angle', range_gate_size=30., **kwargs): self.range_gate_size = range_gate_size - df = self._load(fpath,range_gate,azimuth,elevation) + df = self._load(fpath,range_gate_name,azimuth_name,elevation_name) super().__init__(df, **kwargs) def _load(self,fpath,range_gate,azimuth,elevation): @@ -135,5 +169,5 @@ def _load(self,fpath,range_gate,azimuth,elevation): }) df['range'] *= self.range_gate_size df = df.set_index(['range','azimuth','elevation']).sort_index() - return df + return df.xs(10.5,level='elevation') From 5dfd812c117beb3f1ab12794248c74c8bfc392c4 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 15:57:49 -0600 Subject: [PATCH 093/145] Add input check Remove `.xs()` that was used for testing --- mmctools/measurements/lidar.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index cf85567..115072f 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -23,6 +23,12 @@ def _check_coords(self): for coord in ['range','azimuth','elevation'] ]): if self.verbose: print('3D volumetric scan loaded') + if self.specified_range is not None: + print('Ignoring specified range') + if self.specified_azimuth is not None: + print('Ignoring specified azimuth') + if self.specified_elevation is not None: + print('Ignoring specified elevation') elif 'range' not in self.df.index.names: if self.verbose: print('Vertical scan loaded') if self.specified_range is None: @@ -169,5 +175,5 @@ def _load(self,fpath,range_gate,azimuth,elevation): }) df['range'] *= self.range_gate_size df = df.set_index(['range','azimuth','elevation']).sort_index() - return df.xs(10.5,level='elevation') + return df From f489b839cc777153f07dac12f64e265f9e5dd3ce Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 16:31:00 -0600 Subject: [PATCH 094/145] Create standalone calc_xyz() --- mmctools/measurements/lidar.py | 63 ++++++++++++---------------------- 1 file changed, 22 insertions(+), 41 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 115072f..69f83a3 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -7,44 +7,45 @@ import pandas as pd import xarray as xr +def calc_xyz(df,range=None,azimuth=None,elevation=0.0): + try: + r = df.index.get_level_values('range') + except KeyError: + assert (range is not None), 'need to specify constant value for `range`' + r = range + try: + az = np.radians(270 - df.index.get_level_values('azimuth')) + except ValueError: + assert (range is not None), 'need to specify constant value for `azimuth`' + az = np.radians(azimuth) + try: + el = np.radians(df.index.get_level_values('elevation')) + except ValueError: + el = np.radians(elevation) + x = r * np.cos(az) * np.cos(el) + y = r * np.sin(az) * np.cos(el) + z = r * np.sin(el) + return x,y,z + + class LidarData(object): - def __init__(self,df,range=None,azimuth=None,elevation=None,verbose=True): + def __init__(self,df,verbose=True): """Lidar data described by range, azimuth, and elevation""" self.verbose = verbose self.df = df - self.specified_range = range - self.specified_azimuth = azimuth - self.specified_elevation = elevation self._check_coords() - self._to_xyz() def _check_coords(self): if all([coord in self.df.index.names for coord in ['range','azimuth','elevation'] ]): if self.verbose: print('3D volumetric scan loaded') - if self.specified_range is not None: - print('Ignoring specified range') - if self.specified_azimuth is not None: - print('Ignoring specified azimuth') - if self.specified_elevation is not None: - print('Ignoring specified elevation') elif 'range' not in self.df.index.names: if self.verbose: print('Vertical scan loaded') - if self.specified_range is None: - print('`range` not specified, x/y/z will be invalid') - self.specified_range = 1 elif 'azimuth' not in self.df.index.names: if self.verbose: print('RHI scan loaded') - if self.specified_azimuth is None: - print('`azimuth` not specified, x/y/z may be invalid') - self.specified_azimuth = 0.0 elif 'elevation' not in self.df.index.names: if self.verbose: print('PPI scan loaded') - if self.specified_elevation is None: - if self.verbose: - print('`elevation` not specified, assuming el=0') - self.specified_elevation = 0.0 else: raise IndexError('Unexpected index levels in dataframe: '+str(self.df.index.names)) @@ -55,26 +56,6 @@ def _check_coords(self): self.range_gate_size = dr self.rmax = self.df.index.levels[0][-1] + self.range_gate_size - def _to_xyz(self): - try: - r = self.df.index.get_level_values('range') - except KeyError: - r = self.specified_range - try: - az = np.radians(270 - self.df.index.get_level_values('azimuth')) - except KeyError: - az = np.radians(self.specified_azimuth) - try: - el = np.radians(self.df.index.get_level_values('elevation')) - except KeyError: - el = np.radians(self.specified_elevation) - if not hasattr(self,'x'): - self.x = r * np.cos(az) * np.cos(el) - if not hasattr(self,'y'): - self.y = r * np.sin(az) * np.cos(el) - if not hasattr(self,'z'): - self.z = r * np.sin(el) - @property def r(self): return self.df.index.levels[0] From 9c852c508a243ddfa7e1ffe1e797afde1d679b41 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 18:38:30 -0600 Subject: [PATCH 095/145] Specify azimuth for calc_xyz by met convention --- mmctools/measurements/lidar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 69f83a3..cbc96c8 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -7,6 +7,7 @@ import pandas as pd import xarray as xr + def calc_xyz(df,range=None,azimuth=None,elevation=0.0): try: r = df.index.get_level_values('range') @@ -17,7 +18,7 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0): az = np.radians(270 - df.index.get_level_values('azimuth')) except ValueError: assert (range is not None), 'need to specify constant value for `azimuth`' - az = np.radians(azimuth) + az = np.radians(270 - azimuth) try: el = np.radians(df.index.get_level_values('elevation')) except ValueError: From 1104b5dbe745883da4307afb7bdc373c047dc9f3 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 19:51:33 -0600 Subject: [PATCH 096/145] Fix azimuth angles in calc_xyz() --- mmctools/measurements/lidar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index cbc96c8..9852a1a 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -15,10 +15,10 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0): assert (range is not None), 'need to specify constant value for `range`' r = range try: - az = np.radians(270 - df.index.get_level_values('azimuth')) + az = np.radians(90 - df.index.get_level_values('azimuth')) except ValueError: assert (range is not None), 'need to specify constant value for `azimuth`' - az = np.radians(270 - azimuth) + az = np.radians(90 - azimuth) try: el = np.radians(df.index.get_level_values('elevation')) except ValueError: From 96b415efe5fd2f4b4ab86a3ed1623fb11c6ece65 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sat, 24 Jul 2021 19:51:47 -0600 Subject: [PATCH 097/145] Return middle of range gate --- mmctools/measurements/lidar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 9852a1a..156683e 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -101,7 +101,7 @@ def get_range(self, r): r1 = r + self.range_gate_size if self.verbose: print(f'getting range gate {idx} between {r0} and {r1}') - return self.df.xs(r0, level='range') + return self.df.xs(r0, level='range'), (r0+r1)/2 def get_azimuth(self, az): azs = self.df.index.levels[1] From 8c6e89386b04781a8eaa075441299a6e9e7e8377 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sun, 25 Jul 2021 12:50:10 -0600 Subject: [PATCH 098/145] Add small elev angle option to calc_xyz() --- mmctools/measurements/lidar.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 156683e..41a34a8 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -8,7 +8,8 @@ import xarray as xr -def calc_xyz(df,range=None,azimuth=None,elevation=0.0): +def calc_xyz(df,range=None,azimuth=None,elevation=0.0, + small_elevation_angles=False): try: r = df.index.get_level_values('range') except KeyError: @@ -23,8 +24,11 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0): el = np.radians(df.index.get_level_values('elevation')) except ValueError: el = np.radians(elevation) - x = r * np.cos(az) * np.cos(el) - y = r * np.sin(az) * np.cos(el) + x = r * np.cos(az) + y = r * np.sin(az) + if not small_elevation_angles: + x *= np.cos(el) + y *= np.cos(el) z = r * np.sin(el) return x,y,z From ac678f10715af9e580ce4459967abf175dbf58fa Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sun, 25 Jul 2021 13:16:59 -0600 Subject: [PATCH 099/145] Add RHI, PPI flags --- mmctools/measurements/lidar.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 41a34a8..3948f82 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -38,6 +38,8 @@ def __init__(self,df,verbose=True): """Lidar data described by range, azimuth, and elevation""" self.verbose = verbose self.df = df + self.RHI = False + self.PPI = False self._check_coords() def _check_coords(self): @@ -49,8 +51,10 @@ def _check_coords(self): if self.verbose: print('Vertical scan loaded') elif 'azimuth' not in self.df.index.names: if self.verbose: print('RHI scan loaded') + self.RHI = True elif 'elevation' not in self.df.index.names: if self.verbose: print('PPI scan loaded') + self.PPI = True else: raise IndexError('Unexpected index levels in dataframe: '+str(self.df.index.names)) From 80bf052957200434fdf3be21735f8e55bc79a6e8 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sun, 25 Jul 2021 13:21:21 -0600 Subject: [PATCH 100/145] Rename array names for clarity --- mmctools/measurements/lidar.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 3948f82..ac0a769 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -87,16 +87,16 @@ def get(self, r=None, az=None, el=None): return self.get_elevation(el) def get_range(self, r): - rs = self.df.index.levels[0] + rarray = self.df.index.levels[0] if r < 0: raise ValueError('Invalid range, r < 0') elif r >= self.rmax: raise ValueError(f'Invalid range, r >= {self.rmax}') - if r not in rs: + if r not in rarray: try: - idx = np.where(r < rs)[0][0] - 1 + idx = np.where(r < rarray)[0][0] - 1 except IndexError: - idx = len(rs) - 1 + idx = len(rarray) - 1 r0 = self.df.index.levels[0][idx] r1 = self.rmax else: @@ -104,7 +104,7 @@ def get_range(self, r): r1 = self.df.index.levels[0][idx+1] assert (r >= r0) & (r < r1) else: - idx = list(rs).index(r) + idx = list(rarray).index(r) r0 = r r1 = r + self.range_gate_size if self.verbose: @@ -112,25 +112,25 @@ def get_range(self, r): return self.df.xs(r0, level='range'), (r0+r1)/2 def get_azimuth(self, az): - azs = self.df.index.levels[1] - if az < azs[0]: - raise ValueError(f'Invalid range, az < {azs[0]}') - elif az > azs[-1]: - raise ValueError(f'Invalid range, az > {azs[-1]}') - if az not in azs: - az = azs[np.argmin(np.abs(az - azs))] + azarray = self.df.index.levels[1] + if az < azarray[0]: + raise ValueError(f'Invalid range, az < {azarray[0]}') + elif az > azarray[-1]: + raise ValueError(f'Invalid range, az > {azarray[-1]}') + if az not in azarray: + az = azarray[np.argmin(np.abs(az - azarray))] if self.verbose: print(f'getting nearest azimuth={az} deg') return self.df.xs(az, level='azimuth') def get_elevation(self, el): - els = self.df.index.levels[2] - if el < els[0]: - raise ValueError(f'Invalid range, el < {els[0]}') - elif el > els[-1]: - raise ValueError(f'Invalid range, el > {els[-1]}') - if el not in els: - el = els[np.argmin(np.abs(el - els))] + elarray = self.df.index.levels[2] + if el < elarray[0]: + raise ValueError(f'Invalid range, el < {elarray[0]}') + elif el > elarray[-1]: + raise ValueError(f'Invalid range, el > {elarray[-1]}') + if el not in elarray: + el = elarray[np.argmin(np.abs(el - elarray))] if self.verbose: print(f'getting nearest elevation={el} deg') return self.df.xs(el, level='elevation') From a867d87ebe39fa7fe286bfdb2ae2cf4fbe91dd8f Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sun, 25 Jul 2021 13:41:36 -0600 Subject: [PATCH 101/145] Clean up Galion (Cornell@Perdigao) class, add notes in docstrings --- mmctools/measurements/lidar.py | 43 ++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index ac0a769..d711a6b 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -136,34 +136,41 @@ def get_elevation(self, el): return self.df.xs(el, level='elevation') -class Perdigao(LidarData): - """Galion scanning lidar""" - - def __init__(self, - fpath, - range_gate_name='Range gates', - azimuth_name='Azimuth angle', - elevation_name='Elevation angle', - range_gate_size=30., - **kwargs): - self.range_gate_size = range_gate_size - df = self._load(fpath,range_gate_name,azimuth_name,elevation_name) +class GalionCornellPerdigao(LidarData): + """Data from Galion scanning lidar deployed by Cornell University + at the Perdigao field campaign + + Tested on data retrieved from the UCAR Earth Observing Laboratory + data archive (https://data.eol.ucar.edu/dataset/536.036) retrieved + on 2021-07-24. + """ + + def __init__(self,fpath,load_opts={},**kwargs): + df = self._load(fpath,**load_opts) super().__init__(df, **kwargs) - def _load(self,fpath,range_gate,azimuth,elevation): - """Process a single scan in netcdf format""" + def _load(self, + fpath, + minimum_range=0., + range_gate_size=30., + range_gate_name='Range gates', + azimuth_name='Azimuth angle', + elevation_name='Elevation angle'): + """Process a single scan in netcdf format + + Notes: + - Range gates are stored as integers + - Not all (r,az,el) data points are available + """ ds = xr.open_dataset(fpath) assert (len(ds.dims)==1) and ('y' in ds.dims) - r = np.unique(ds[range_gate]) * self.range_gate_size - az = np.unique(ds[azimuth]) - el = np.unique(ds[elevation]) df = ds.to_dataframe() df = df.rename(columns={ 'Range gates': 'range', 'Azimuth angle': 'azimuth', 'Elevation angle': 'elevation', }) - df['range'] *= self.range_gate_size + df['range'] = minimum_range + df['range']*range_gate_size df = df.set_index(['range','azimuth','elevation']).sort_index() return df From b7a6a789680c9852f385d47dec48d3b395321600 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sun, 25 Jul 2021 14:10:25 -0600 Subject: [PATCH 102/145] Update usage of r, which now corresponds to the range gate centers Code clean up, add comments to get*() functions --- mmctools/measurements/lidar.py | 66 ++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index d711a6b..9e4d053 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -40,9 +40,10 @@ def __init__(self,df,verbose=True): self.df = df self.RHI = False self.PPI = False - self._check_coords() + self._check_data() - def _check_coords(self): + def _check_data(self): + # check coordinates if all([coord in self.df.index.names for coord in ['range','azimuth','elevation'] ]): @@ -57,13 +58,15 @@ def _check_coords(self): self.PPI = True else: raise IndexError('Unexpected index levels in dataframe: '+str(self.df.index.names)) - - dr = self.df.index.levels[0][1] - self.df.index.levels[0][0] + # check ranges + rarray = self.df.index.levels[0] + dr = rarray[1] - rarray[0] if hasattr(self, 'range_gate_size'): assert self.range_gate_size == dr else: self.range_gate_size = dr - self.rmax = self.df.index.levels[0][-1] + self.range_gate_size + self.rmin = rarray[0] - self.range_gate_size/2 + self.rmax = rarray[-1] + self.range_gate_size/2 @property def r(self): @@ -79,39 +82,51 @@ def el(self): # slicers def get(self, r=None, az=None, el=None): + """Wrapper for get_range, get_azimuth, and get_elevation()""" if r is not None: return self.get_range(r) elif az is not None: return self.get_azimuth(az) elif el is not None: return self.get_elevation(el) + else: + print('Specify r, az, or el') def get_range(self, r): - rarray = self.df.index.levels[0] - if r < 0: - raise ValueError('Invalid range, r < 0') + """Get range gate containing request range + + Returns a copy of a multiindex dataframe with (r,az,el) indices + where r is center of the range gate. + """ + rarray = self.df.index.levels[0] # center of each range gate + if r < self.rmin: + raise ValueError(f'Invalid range, r < {self.rmin}') elif r >= self.rmax: raise ValueError(f'Invalid range, r >= {self.rmax}') if r not in rarray: - try: - idx = np.where(r < rarray)[0][0] - 1 - except IndexError: - idx = len(rarray) - 1 - r0 = self.df.index.levels[0][idx] - r1 = self.rmax - else: - r0 = self.df.index.levels[0][idx] - r1 = self.df.index.levels[0][idx+1] - assert (r >= r0) & (r < r1) + right_edges = rarray + self.range_gate_size/2 + idx = np.where(r < right_edges)[0][0] + rsel = rarray[idx] + r0 = rsel - self.range_gate_size/2 + r1 = rsel + self.range_gate_size/2 + assert (r >= r0) & (r < r1), f'{r} is not between {r0} and {r1}' + if self.verbose: + print(f'getting nearest range gate {idx} between {r0} and {r1}') else: + rsel = r idx = list(rarray).index(r) - r0 = r - r1 = r + self.range_gate_size - if self.verbose: - print(f'getting range gate {idx} between {r0} and {r1}') - return self.df.xs(r0, level='range'), (r0+r1)/2 + r0 = r - self.range_gate_size/2 + r1 = r + self.range_gate_size/2 + if self.verbose: + print(f'getting range gate {idx} between {r0} and {r1}') + #return self.df.xs(rsel, level='range') + return self.df.loc[(rsel,slice(None),slice(None)),:] def get_azimuth(self, az): + """Get requested azimuth (i.e., an RHI slice) + + Returns a copy of a multiindex dataframe with (r,el) indices. + """ azarray = self.df.index.levels[1] if az < azarray[0]: raise ValueError(f'Invalid range, az < {azarray[0]}') @@ -124,6 +139,10 @@ def get_azimuth(self, az): return self.df.xs(az, level='azimuth') def get_elevation(self, el): + """Get requested elevation (i.e., a PPI slice) + + Returns a copy of a multiindex dataframe with (r,az) indices. + """ elarray = self.df.index.levels[2] if el < elarray[0]: raise ValueError(f'Invalid range, el < {elarray[0]}') @@ -171,6 +190,7 @@ def _load(self, 'Elevation angle': 'elevation', }) df['range'] = minimum_range + df['range']*range_gate_size + df['range'] += range_gate_size/2 # shift to center of range gate df = df.set_index(['range','azimuth','elevation']).sort_index() return df From 126d5f71020254c35d0def3a666aefc52533fcbe Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Sun, 25 Jul 2021 14:22:27 -0600 Subject: [PATCH 103/145] Update small elevation angle assumption for z --- mmctools/measurements/lidar.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 9e4d053..4aa3795 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -24,12 +24,14 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0, el = np.radians(df.index.get_level_values('elevation')) except ValueError: el = np.radians(elevation) - x = r * np.cos(az) - y = r * np.sin(az) - if not small_elevation_angles: - x *= np.cos(el) - y *= np.cos(el) - z = r * np.sin(el) + if small_elevation_angles: + x = r * np.cos(az) + y = r * np.sin(az) + z = r * el + else: + x = r * np.cos(az)* np.cos(el) + y = r * np.sin(az)* np.cos(el) + z = r * np.sin(el) return x,y,z From 2dbead73ee89f95800c83c90f7de5e2e9e82095c Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Mon, 26 Jul 2021 00:19:38 -0600 Subject: [PATCH 104/145] Add class for Cornell Galion in PEIWEE --- mmctools/measurements/lidar.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 4aa3795..91029e4 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -17,12 +17,12 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0, r = range try: az = np.radians(90 - df.index.get_level_values('azimuth')) - except ValueError: + except KeyError: assert (range is not None), 'need to specify constant value for `azimuth`' az = np.radians(90 - azimuth) try: el = np.radians(df.index.get_level_values('elevation')) - except ValueError: + except KeyError: el = np.radians(elevation) if small_elevation_angles: x = r * np.cos(az) @@ -187,12 +187,34 @@ def _load(self, assert (len(ds.dims)==1) and ('y' in ds.dims) df = ds.to_dataframe() df = df.rename(columns={ - 'Range gates': 'range', - 'Azimuth angle': 'azimuth', - 'Elevation angle': 'elevation', + range_gate_name: 'range', + azimuth_name: 'azimuth', + elevation_name: 'elevation', }) df['range'] = minimum_range + df['range']*range_gate_size df['range'] += range_gate_size/2 # shift to center of range gate df = df.set_index(['range','azimuth','elevation']).sort_index() return df + +class GalionCornellPEIWEE(LidarData): + """Data from Galion scanning lidar deployed by Cornell University + at the Prince Edward Island Wind Energy Experiment + """ + + def __init__(self,fpath,load_opts={},**kwargs): + df = self._load(fpath,**load_opts) + super().__init__(df, **kwargs) + + def _load(self, + fpath, + minimum_range=0., + range_gate_size=30.): + """Process a single scan in netcdf format + """ + df = pd.read_csv(fpath) + df['range'] = minimum_range + df['range_gate']*range_gate_size + df['range'] += range_gate_size/2 # shift to center of range gate + df = df.set_index(['range','azimuth','elevation']).sort_index() + return df + From 2b484cd5c284065e7915506076c0dc0396b18751 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 27 Jul 2021 10:18:21 -0600 Subject: [PATCH 105/145] Add docstring to calc_xyz() --- mmctools/measurements/lidar.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 91029e4..b3ca967 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -10,6 +10,21 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0, small_elevation_angles=False): + """Transform scan data with range (range gate center), azimuth, and + elevation indices to Cartesian x,y,z coordinates + + Parameters + ---------- + range : optional + Needed for vertical scan data without a range coordinate + azimuth : optional + Needed for RHI scan data without an azimuth coordinate + elevation : optional + Needed for PPI scan data without an elevation coordinate + small_elevation_angles : optional + Invoke small angle approximation, i.e., cos(x)~1 and sin(x)~x + """ + try: r = df.index.get_level_values('range') except KeyError: From 632be4bbf63b2f0cc7939cc0c67be2925d48d8db Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 27 Jul 2021 10:31:06 -0600 Subject: [PATCH 106/145] Update calc_xyz() docstring --- mmctools/measurements/lidar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index b3ca967..aa908c3 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -16,11 +16,11 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0, Parameters ---------- range : optional - Needed for vertical scan data without a range coordinate + Needed for vertical scan data without a range coordinate [m] azimuth : optional - Needed for RHI scan data without an azimuth coordinate + Needed for RHI scan data without an azimuth coordinate [deg] elevation : optional - Needed for PPI scan data without an elevation coordinate + Needed for PPI scan data without an elevation coordinate [deg] small_elevation_angles : optional Invoke small angle approximation, i.e., cos(x)~1 and sin(x)~x """ From bc6b491671a37556db860e369d9a653a0d1650b5 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 3 Aug 2021 11:49:43 -0600 Subject: [PATCH 107/145] Optionally init GalionCornellPEIWEE with df --- mmctools/measurements/lidar.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index aa908c3..38b62f8 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -227,7 +227,10 @@ def _load(self, range_gate_size=30.): """Process a single scan in netcdf format """ - df = pd.read_csv(fpath) + if isinstance(fpath, pd.DataFrame): + df = fpath.copy() + else: + df = pd.read_csv(fpath) df['range'] = minimum_range + df['range_gate']*range_gate_size df['range'] += range_gate_size/2 # shift to center of range gate df = df.set_index(['range','azimuth','elevation']).sort_index() From 5c98c8238b497bdd9c655695ef6c47410bb67f01 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Tue, 3 Aug 2021 12:28:22 -0600 Subject: [PATCH 108/145] Add option to filter by range, intensity --- mmctools/measurements/lidar.py | 35 +++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 38b62f8..6e9b999 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -217,9 +217,15 @@ class GalionCornellPEIWEE(LidarData): at the Prince Edward Island Wind Energy Experiment """ - def __init__(self,fpath,load_opts={},**kwargs): + def __init__(self,fpath,load_opts={}, + range_gates=None, + ranges=None, + intensity_range=(None,None), + **kwargs): df = self._load(fpath,**load_opts) super().__init__(df, **kwargs) + self._filter_by_range(range_gates=range_gates, ranges=ranges) + self._filter_by_intensity(*intensity_range) def _load(self, fpath, @@ -236,3 +242,30 @@ def _load(self, df = df.set_index(['range','azimuth','elevation']).sort_index() return df + def _filter_by_range(self,range_gates=None,ranges=None): + assert range_gates or ranges, 'Specify range_gates or ranges' + allranges = self.df.index.get_level_values('range') + if range_gates: + assert isinstance(range_gates, (tuple,list)), \ + 'Specify range_gates as (min_gate, max_gate)' + if range_gates[0] is not None: + minrange = allranges[range_gates[0]] + self.df.loc[allranges <= minrange,:] = np.nan + if range_gates[1] is not None: + maxrange = allranges[range_gates[1]] + self.df.loc[allranges >= maxrange,:] = np.nan + elif ranges: + assert isinstance(ranges, (tuple,list)), \ + 'Specify ranges as (min_range, max_range)' + if ranges[0] is not None: + self.df.loc[allranges < ranges[0],:] = np.nan + if ranges[1] is not None: + self.df.loc[allranges > ranges[1],:] = np.nan + + def _filter_by_intensity(self,min_intensity,max_intensity): + if min_intensity is not None: + self.df.loc[self.df['intensity']max_intensity, :] = np.nan + + From d8618e90ee7127e7bf9b441387c9558e06c21d4c Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 4 Aug 2021 01:35:25 -0600 Subject: [PATCH 109/145] Add measurements.lidar.estimate_mean_wind() --- mmctools/measurements/lidar.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/mmctools/measurements/lidar.py b/mmctools/measurements/lidar.py index 6e9b999..1d19a74 100644 --- a/mmctools/measurements/lidar.py +++ b/mmctools/measurements/lidar.py @@ -50,6 +50,44 @@ def calc_xyz(df,range=None,azimuth=None,elevation=0.0, return x,y,z +def estimate_mean_wind(doppler,elevation=None): + """Estimate mean wind speed and direction given an arc scan + + Parameters + ---------- + doppler : pd.DataFrame or pd.Series + Radial velocity with range, azimuth, and (optionally) elevation + indices; if a single elevation cannot be inferred from the data, + `elevation` must be specified + elevation : float, optional + Scan elevation angle [deg] + """ + try: + el = doppler.index.get_level_values('elevation') + except KeyError: + assert (elevation is not None), 'PPI scan, need to specify elevation' + el = elevation + else: + assert np.all(el == el[0]) + el = np.radians(el) + az = np.radians(doppler.index.get_level_values('azimuth')) + comp1 = np.cos(el)*np.sin(az) + comp2 = np.cos(el)*np.cos(az) + missingvals = doppler.isna() + if np.any(missingvals): + comp1 = comp1[~missingvals] + comp2 = comp2[~missingvals] + doppler = doppler.copy() + doppler = doppler.loc[~missingvals] + LHS = np.stack([comp1, comp2], axis=-1) + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False).fit(LHS, doppler) + U0,V0 = reg.coef_ + Vmag = np.sqrt(U0**2 + V0**2) + beta = 180. + np.degrees(np.arctan2(U0,V0)) # wind dir follows meteorological convention + return Vmag, beta + + class LidarData(object): def __init__(self,df,verbose=True): """Lidar data described by range, azimuth, and elevation""" From 348dc2db01ae6be0021d5b405557da91af1b070e Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 4 Aug 2021 18:39:43 -0600 Subject: [PATCH 110/145] Add calculation of topographic shelter Sx --- mmctools/coupling/terrain.py | 80 ++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 899752c..ec17a2c 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -624,3 +624,83 @@ def vrm_filt(x): else: return vrm + +def calcSx (xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbose=False): + ''' + Sx is a measure of topographic shelter or exposure relative to a particular + wind direction. Calculates a whole map for all points (xi, yi) in the domain. + For each (xi, yi) pair, it uses all v points (xv, yv) upwind of (xi, yi) in + the A wind direction, up to dmax. + + Winstral, A., Marks D. "Simulating wind fields and snow redistribution using + terrain-based parameters to model snow accumulation and melt over a semi- + arid mountain catchment" Hydrol. Process. 16, 3585–3603 (2002) + + Usage + ===== + xx, yy : array + meshgrid arrays of the region extent coordinates. + zagl: array + Elevation map of the region + A: float + Wind direction (deg, wind direction convention) + dmax: float + Upwind extent of the search + method: string + griddata interpolation method. Options are 'nearest', 'linear', 'cubic'. + Function is slow if not `nearest`. + propagateNaN: bool + If method != nearest, upwind posititions that lie outside the domain bounds receive NaN + ''' + + from scipy import interpolate + + # create empty output Sx array + Sx = np.empty(np.shape(zagl)); Sx[:,:] = np.nan + + # get resolution (assumes uniform resolution) + res = xx[1,0] - xx[0,0] + npoints = 1+int(dmax/res) + if dmax < res: + raise ValueError('dmax needs to be larger or equal to the resolution of the grid') + + # change angle notation + ang = np.deg2rad(270-A) + + # array for interpolation using griddata + points = np.array( (xx.flatten(), yy.flatten()) ).T + values = zagl.flatten() + + for i, xi in enumerate(xx[:,0]): + print(f'Processing row {i+1}/{len(xx)} ', end='\r') + for j, yi in enumerate(yy[0,:]): + + # limits of the line where Sx will be calculated on (minus bc it's upwind) + xf = xi - dmax*np.cos(ang) + yf = yi - dmax*np.sin(ang) + xline = np.around(np.linspace(xi, xf, num=npoints), decimals=4) + yline = np.around(np.linspace(yi, yf, num=npoints), decimals=4) + + # interpolate points upstream (xi, yi) along angle ang + elev = interpolate.griddata( points, values, (xline,yline), method=method ) + + # elevation of (xi, yi), for convenience + elevi = elev[0] + + if propagateNaN: + Sx[i,j] = np.amax(np.rad2deg( np.arctan( (elev[1:] - elevi)/(((xline[1:]-xi)**2 + (yline[1:]-yi)**2)**0.5) ) )) + else: + Sx[i,j] = np.nanmax(np.rad2deg( np.arctan( (elev[1:] - elevi)/(((xline[1:]-xi)**2 + (yline[1:]-yi)**2)**0.5) ) )) + + if verbose: print(f'Max angle is {Sx:.4f} degrees') + + return Sx + +def calcSxmean (xx, yy, zagl, A, dmax, method='nearest', verbose=False): + + Asweep = np.linspace(A-15, A+15, 7)%360 + Sxmean = np.mean([calcSx(xx, yy, zagl, a, dmax, method, verbose=verbose) for a in Asweep ], axis=0) + + return Sxmean + + From cbf3901a13f15f3b5615a149501b68f39d0e93c9 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Thu, 5 Aug 2021 08:34:32 -0600 Subject: [PATCH 111/145] Add Sb calculation --- mmctools/coupling/terrain.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index ec17a2c..294083d 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -704,3 +704,50 @@ def calcSxmean (xx, yy, zagl, A, dmax, method='nearest', verbose=False): return Sxmean +def calcSb (xx, yy, zagl, A, sepdist=60): + ''' + Sb is a measure of upwind slope break and can be used to delineate zones of + possible flow separation. This function follows the definition of Sx0 from + the reference listed below and uses 1000m separation. + + Winstral, A., Marks D. "Simulating wind fields and snow redistribution using + terrain-based parameters to model snow accumulation and melt over a semi- + arid mountain catchment" Hydrol. Process. 16, 3585–3603 (2002) + + Usage + ===== + xx, yy : array + meshgrid arrays of the region extent coordinates. + zagl: array + Elevation map of the region + A: float + Wind direction (deg, wind direction convention) + sepdist : float, default 60 + Separation between between two regional Sx calculations. + Suggested value: 60 m. + ''' + + from scipy import interpolate + + # local Sx + Sx1 = calcSx(xx, yy, zagl, A, dmax=sepdist) + + # outlying Sx. Computing it at (xo, yo), and not at (xi, yi) + xxo = xx - sepdist*np.cos(np.deg2rad(270-A)) + yyo = yy - sepdist*np.sin(np.deg2rad(270-A)) + points = np.array( (xx.flatten(), yy.flatten()) ).T + values = zagl.flatten() + zaglo = interpolate.griddata( points, values, (xxo,yyo), method='linear' ) + Sx0 = calcSx(xxo, yyo, zaglo, A, dmax=1000) + + Sb = Sx1 - Sx0 + + return Sb + +def calcSbmean (xx, yy, zagl, A, sepdist): + + Asweep = np.linspace(A-15, A+15, 7)%360 + Sbmean = np.mean([calcSb(xx, yy, zagl, a, sepdist) for a in Asweep ], axis=0) + + return Sbmean + From a63cb93ecacfb6f5102b6772deb1461633c5e1e7 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Thu, 5 Aug 2021 08:35:55 -0600 Subject: [PATCH 112/145] Add topographic position index calculation --- mmctools/coupling/terrain.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 294083d..4327075 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -751,3 +751,27 @@ def calcSbmean (xx, yy, zagl, A, sepdist): return Sbmean + +def calcTPI (xx, yy, zagl, r): + ''' + Topographic Position Index + + Reu, J, et al. Application of the topographic position index to heterogeneous + landscapes. Geomorphology, 186, 39-49 (2013) + ''' + from scipy.signal import convolve2d + + # get resolution (assumes uniform resolution) + res = xx[1,0] - xx[0,0] + rpoints = int(r/res) + if r < res: + raise ValueError('Averaging radium needs to be larger the resolution of the grid') + + y,x = np.ogrid[-rpoints:rpoints+1, -rpoints:rpoints+1] + kernel = x**2+y**2 <= rpoints**2 + + zaglmean = convolve2d(zagl, kernel*1, mode='same', boundary='fill', fillvalue=0) + zaglmean = zaglmean/np.sum(kernel*1) + + return zagl - zaglmean + From 7987d627c45780c1804b70d5208e3676ba14cf0d Mon Sep 17 00:00:00 2001 From: ewquon Date: Mon, 9 Aug 2021 12:35:01 -0600 Subject: [PATCH 113/145] Minor cleanup --- mmctools/coupling/terrain.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 4327075..f2aebfa 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -625,7 +625,7 @@ def vrm_filt(x): return vrm -def calcSx (xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbose=False): +def calcSx(xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbose=False): ''' Sx is a measure of topographic shelter or exposure relative to a particular wind direction. Calculates a whole map for all points (xi, yi) in the domain. @@ -650,7 +650,7 @@ def calcSx (xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbos griddata interpolation method. Options are 'nearest', 'linear', 'cubic'. Function is slow if not `nearest`. propagateNaN: bool - If method != nearest, upwind posititions that lie outside the domain bounds receive NaN + If method != nearest, upwind positions that lie outside the domain bounds receive NaN ''' from scipy import interpolate @@ -696,7 +696,7 @@ def calcSx (xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbos return Sx -def calcSxmean (xx, yy, zagl, A, dmax, method='nearest', verbose=False): +def calcSxmean(xx, yy, zagl, A, dmax, method='nearest', verbose=False): Asweep = np.linspace(A-15, A+15, 7)%360 Sxmean = np.mean([calcSx(xx, yy, zagl, a, dmax, method, verbose=verbose) for a in Asweep ], axis=0) @@ -704,7 +704,7 @@ def calcSxmean (xx, yy, zagl, A, dmax, method='nearest', verbose=False): return Sxmean -def calcSb (xx, yy, zagl, A, sepdist=60): +def calcSb(xx, yy, zagl, A, sepdist=60): ''' Sb is a measure of upwind slope break and can be used to delineate zones of possible flow separation. This function follows the definition of Sx0 from @@ -744,7 +744,7 @@ def calcSb (xx, yy, zagl, A, sepdist=60): return Sb -def calcSbmean (xx, yy, zagl, A, sepdist): +def calcSbmean(xx, yy, zagl, A, sepdist): Asweep = np.linspace(A-15, A+15, 7)%360 Sbmean = np.mean([calcSb(xx, yy, zagl, a, sepdist) for a in Asweep ], axis=0) @@ -752,7 +752,7 @@ def calcSbmean (xx, yy, zagl, A, sepdist): return Sbmean -def calcTPI (xx, yy, zagl, r): +def calcTPI(xx, yy, zagl, r): ''' Topographic Position Index From e17a15ab11db77d3507ae437c042181f99acf07a Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 21 Sep 2021 15:03:39 -0600 Subject: [PATCH 114/145] Adjusting TRI/VRM calcs --- mmctools/helper_functions.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 3d36f6a..d10ca8b 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1322,7 +1322,7 @@ def tri_filt(x): return tri -def calcVRM(hgt,window=None,footprint=None,slope_zscale=1.0,return_slope=False): +def calcVRM(hgt,res,window=None,footprint=None,slope_zscale=1.0,return_slope=False): ''' Vector Ruggedness Measure Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). @@ -1355,15 +1355,17 @@ def calcVRM(hgt,window=None,footprint=None,slope_zscale=1.0,return_slope=False): # Get slope and aspect: hgt_rd = rd.rdarray(hgt, no_data=-9999) rd.FillDepressions(hgt_rd, in_place=True) - slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun', zscale=slope_zscale) + #slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun', zscale=slope_zscale) + zscale = 1/res + slope = rd.TerrainAttribute(hgt_rd, attrib='slope_degrees',zscale=zscale) aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') # Calculate vectors: vrm = np.zeros((ny,nx)) - rugz = np.cos(slope*np.pi/180.0) - rugdxy = np.sin(slope*np.pi/180.0) - rugx = rugdxy*np.cos(aspect*np.pi/180.0) - rugy = rugdxy*np.sin(aspect*np.pi/180.0) + rugz = np.cos(np.deg2rad(slope)) + rugdxy = np.sin(np.deg2rad(slope)) + rugx = rugdxy*np.cos(np.deg2rad(aspect)) + rugy = rugdxy*np.sin(np.deg2rad(aspect)) def vrm_filt(x): return(sum(x)**2) From dc39816271e8e7c1869ac45e1cd48337141435bd Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 21 Sep 2021 15:06:50 -0600 Subject: [PATCH 115/145] Adding class to calculate eta levels Class will allow for user-specified heights to be converted to eta levels (given some other parameters), if the heights do not reach the model top, then the code can fill in levels (using the same formulation that WRF uses), and if there are any jumps in d(eta) the user can smooth the levels. Smoothing will change the eta levels, so if there is constant spacing at some point, the spline may adjust it so that it is not quite constant anymore. Alternatively, if the user has eta levels already but wants to smooth out any kinks, the code accepts eta levels and then the user can simply call the 'smooth_eta_levels function to generate a smoothed list. --- mmctools/wrf/preprocessing.py | 175 ++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index caf1c61..22db9ca 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1553,3 +1553,178 @@ def _write_new_file(self,met_file): print('File exists... replacing') os.remove(new_file) new.to_netcdf(new_file) + +class create_eta_levels(): + ''' + Generate a list of eta levels for WRF simulations. Core of the eta level code + comes from Tim Juliano of NCAR. Alternatively, the user can provide a list of + eta levels and use the smooth_eta_levels feature. + + Usage: + ===== + levels : list or array + The list of levels you want to convert to eta levels + surface_temp : float + average temperature at the surface + pres_top : float + pressure at the top of the domain + height_top : float + height of the top of the domain + p0 :float + reference pressure / pressure at the surface + n_total_levels : int + number of total levels. If levels does not reach the model + top, then this needs to be specified so that the code + knows how many more levels to generate + fill_to_top : boolean + if len(levels) != n_total_levels then this tells the code + to fill to the top or not + smooth_eta : boolean + If true, this will use a spline interpolation on the d(eta) + levels and re-normalize between 1.0 and 0.0 so that there + is a smooth transition between the specified levels and the + filled levels. + + Examples: + 1. Levels specified to the model top: + + eta_levels = generate_eta_levels(levels=np.arange(0,4000.1,20.0), + pres_top=62500.0, + surface_temp=290.0, + height_top=4000, + n_total_levels=201, + smooth_eta=False + ) + + + 2. Only specify lower levels and let the program fill the rest (no smoothing): + + eta_levels = generate_eta_levels(levels=np.linspace(0,1000,50), + pres_top=10000.0, + surface_temp=282.72, + height_top=16229.028, + n_total_levels=88, + smooth_eta=False + ) + + 3. Smooth the eta levels so there are no harsh jumps in d(eta): + + eta_levels = generate_eta_levels(levels=np.linspace(0,1000,50), + pres_top=10000.0, + surface_temp=282.72, + height_top=16229.028, + n_total_levels=88, + smooth_eta=True + ) + + ''' + + def __init__(self, levels=None, + eta_levels=None, + surface_temp=290, + pres_top=None, + height_top=14417.41, + p0=100000.0, + fill_to_top=True, + n_total_levels=None): + + if eta_levels is not None: + self.eta_levels = eta_levels + else: + if (levels is None) or ((type(levels) is not list) and (type(levels) is not np.array) and (type(levels) is not np.ndarray)): + print('Please specify levels in list or array') + return + else: + if not (all((isinstance(z, float) or isinstance(z, int) or isinstance(z, np.int64)) for z in levels)): + print('Levels must be of type float or integer') + return + if type(levels) is list: + levels = np.asarray(levels) + + self.levels = levels + + if n_total_levels < len(self.levels): + print('Setting n_total_levels to be len(levels).') + n_total_levels = len(levels) + + pressure = self._pressure_calc(surface_temp,height_top,p0) + if pres_top is None: + pres_top = pressure[-1] + + self.eta_levels = self._eta_level_calc(pressure,pres_top,height_top,p0,n_total_levels,fill_to_top) + + + def _pressure_calc(self,surface_temp, + height_top, + p0, + pres_calc_option=1): + gas_constant_dry_air = 287.06 + gravity = 9.80665 + M = 0.0289644 + universal_gas_constant = 8.3144598 + + if pres_calc_option == 1: + pressure = p0*np.exp((-gravity*self.levels)/gas_constant_dry_air/surface_temp) + elif pres_calc_option == 2: + pressure = p0*np.exp((-gravity*M*self.levels/(universal_gas_constant*surface_temp))) + else: + print('pres_calc_option = {} is not a valid option. Please select 1 or 2'.format(pres_calc_option)) + + return(pressure) + + def _eta_level_calc(self,pressure, + pres_top, + height_top, + p0, + n_total_levels, + fill_to_top): + + eta_levels = np.zeros(n_total_levels) + + eta_levels[:len(self.levels)] = (pressure-pres_top)/(p0-pres_top) + + if np.max(self.levels) < height_top: + if (n_total_levels is None) or (n_total_levels <= len(self.levels)): + print('Insufficient number of levels to reach model top.') + print('Height top: {}, top of specified levels: {}, number of levels: {}'.format( + height_top,self.levels[-1],n_total_levels)) + print('Must specify n_total_levels to complete eta_levels to model top') + return + remaining_levels = n_total_levels - len(self.levels) + + eta_levels_top = np.zeros(remaining_levels+2) + z_scale = 0.4 + for k in range(1,remaining_levels+2): + kind = k-1 + eta_levels_top[kind] = (np.exp(-(k-1)/float(n_total_levels)/z_scale) - np.exp(-1./z_scale))/ (1.-np.exp(-1./z_scale)) + eta_levels_top -= eta_levels_top[-2] + eta_levels_top = eta_levels_top[:-1] + eta_levels_top /= np.max(eta_levels_top) + eta_levels_top *= eta_levels[len(self.levels)-1] + + eta_levels[len(self.levels):] = eta_levels_top[1:] + + return(eta_levels) + + def smooth_eta_levels(self, + smooth_fact=7e-4, + smooth_degree=2): + + eta_levels = self.eta_levels + deta_x = np.arange(0,len(eta_levels)-1) + deta = eta_levels[1:] - eta_levels[:-1] + + spl = UnivariateSpline(deta_x,deta,k=smooth_degree) + + spl.set_smoothing_factor(smooth_fact) + deta_x = np.arange(0,len(eta_levels)-1) + new_deta = spl(deta_x) + + + final_eta_levels = np.ones(len(new_deta)+1) + for ee,eta in enumerate(final_eta_levels[1:]): + final_eta_levels[ee+1] = 1 + sum(new_deta[:ee+1]) + final_eta_levels -= min(final_eta_levels) + final_eta_levels /= max(final_eta_levels) + + return(final_eta_levels) From a15e36f0d62cb18034bf68d5120caeab8709dc2d Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 21 Sep 2021 15:20:07 -0600 Subject: [PATCH 116/145] Changing the naming convention to match others --- mmctools/wrf/preprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 22db9ca..863c0b4 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1554,7 +1554,7 @@ def _write_new_file(self,met_file): os.remove(new_file) new.to_netcdf(new_file) -class create_eta_levels(): +class CreateEtaLevels(): ''' Generate a list of eta levels for WRF simulations. Core of the eta level code comes from Tim Juliano of NCAR. Alternatively, the user can provide a list of From 65702a780a222745366fd7f9abb6e17070c4c1c7 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 22 Sep 2021 07:33:49 -0600 Subject: [PATCH 117/145] Removing TRI and VRM since they are redundant --- mmctools/helper_functions.py | 109 ----------------------------------- 1 file changed, 109 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index b0c1059..fda99c3 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1280,112 +1280,3 @@ def calc_spectra(data, else: psd_f = psd_level.combine_first(psd_f) return(psd_f) - -def calcTRI(hgt,window=None,footprint=None): - ''' - Terrain Ruggedness Index - Riley, S. J., DeGloria, S. D., & Elliot, R. (1999). Index that - quantifies topographic heterogeneity. intermountain Journal - of sciences, 5(1-4), 23-27. - - hgt : array - Array of heights over which TRI will be calculated - window : int - Length of window in x and y direction. Must be odd. - ''' - from scipy.ndimage.filters import generic_filter - - # Window setup: - if footprint is not None: - assert window is None, 'Must specify either window or footprint' - window = np.shape(footprint)[0] - - assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' - Hwindow = int(np.floor(window/2)) - - # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): - hgt = hgt.data - assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) - - ny,nx = np.shape(hgt) - - def tri_filt(x): - middle_ind = int(len(x)/2) - return((sum((x - x[middle_ind])**2.0))**0.5) - - if footprint is None: - tri = generic_filter(hgt,tri_filt, size = (window,window)) - else: - tri = generic_filter(hgt,tri_filt, footprint=footprint) - - return tri - - -def calcVRM(hgt,res,window=None,footprint=None,slope_zscale=1.0,return_slope=False): - ''' - Vector Ruggedness Measure - Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). - Quantifying landscape ruggedness for animal habitat analysis: - a case study using bighorn sheep in the Mojave Desert. The - Journal of wildlife management, 71(5), 1419-1426. - - hgt : array - Array of heights over which TRI will be calculated - window : int - Length of window in x and y direction. Must be odd. - ''' - import richdem as rd - from scipy.ndimage.filters import generic_filter - - # Window setup: - if footprint is not None: - assert window is None, 'Must specify either window or footprint' - window = np.shape(footprint)[0] - - assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' - Hwndw = int(np.floor(window/2)) - - # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): - hgt = hgt.data - assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) - ny,nx = np.shape(hgt) - - # Get slope and aspect: - hgt_rd = rd.rdarray(hgt, no_data=-9999) - rd.FillDepressions(hgt_rd, in_place=True) - #slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun', zscale=slope_zscale) - zscale = 1/res - slope = rd.TerrainAttribute(hgt_rd, attrib='slope_degrees',zscale=zscale) - aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') - - # Calculate vectors: - vrm = np.zeros((ny,nx)) - rugz = np.cos(np.deg2rad(slope)) - rugdxy = np.sin(np.deg2rad(slope)) - rugx = rugdxy*np.cos(np.deg2rad(aspect)) - rugy = rugdxy*np.sin(np.deg2rad(aspect)) - - def vrm_filt(x): - return(sum(x)**2) - - if footprint is None: - vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) - vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) - vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) - else: - vrmX = generic_filter(rugx,vrm_filt, footprint=footprint) - vrmY = generic_filter(rugy,vrm_filt, footprint=footprint) - vrmZ = generic_filter(rugz,vrm_filt, footprint=footprint) - - - if footprint is not None: - num_points = len(footprint[footprint != 0.0]) - else: - num_points = float(window**2) - vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/num_points - if return_slope: - return vrm,slope - else: - return vrm From 61693bc01946e60f8366a02316416b26ee17dae9 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 22 Sep 2021 07:34:57 -0600 Subject: [PATCH 118/145] Adding function to print eta levels for WRF This makes it easy to copy/paste the eta levels into a namelist --- mmctools/wrf/preprocessing.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 863c0b4..c2457fd 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -6,6 +6,7 @@ import glob import xarray as xr from mmctools.helper_functions import get_nc_file_times +from scipy.interpolate import UnivariateSpline def prompt(s): if sys.version_info[0] < 3: @@ -1728,3 +1729,19 @@ def smooth_eta_levels(self, final_eta_levels /= max(final_eta_levels) return(final_eta_levels) + + + def print_eta_levels(self,ncols=4): + count = 0 + line = '' + print('{} levels'.format(len(self.eta_levels))) + for kk,eta in enumerate(self.eta_levels): + line += '{0:6.5f}, '.format(eta) + count+=1 + if count == ncols: + #line += '\n' + print(line) + count = 0 + line = '' + if line != '': + print(line) From 26fbea76244b80cefb3979464e633709da24eb0b Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 22 Sep 2021 07:36:00 -0600 Subject: [PATCH 119/145] Removing redundant TRI and VRM calculations These are already in mmctools.coupling.terrain --- mmctools/helper_functions.py | 108 ----------------------------------- 1 file changed, 108 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index b0c1059..f3c75f2 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1281,111 +1281,3 @@ def calc_spectra(data, psd_f = psd_level.combine_first(psd_f) return(psd_f) -def calcTRI(hgt,window=None,footprint=None): - ''' - Terrain Ruggedness Index - Riley, S. J., DeGloria, S. D., & Elliot, R. (1999). Index that - quantifies topographic heterogeneity. intermountain Journal - of sciences, 5(1-4), 23-27. - - hgt : array - Array of heights over which TRI will be calculated - window : int - Length of window in x and y direction. Must be odd. - ''' - from scipy.ndimage.filters import generic_filter - - # Window setup: - if footprint is not None: - assert window is None, 'Must specify either window or footprint' - window = np.shape(footprint)[0] - - assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' - Hwindow = int(np.floor(window/2)) - - # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): - hgt = hgt.data - assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) - - ny,nx = np.shape(hgt) - - def tri_filt(x): - middle_ind = int(len(x)/2) - return((sum((x - x[middle_ind])**2.0))**0.5) - - if footprint is None: - tri = generic_filter(hgt,tri_filt, size = (window,window)) - else: - tri = generic_filter(hgt,tri_filt, footprint=footprint) - - return tri - - -def calcVRM(hgt,res,window=None,footprint=None,slope_zscale=1.0,return_slope=False): - ''' - Vector Ruggedness Measure - Sappington, J. M., Longshore, K. M., & Thompson, D. B. (2007). - Quantifying landscape ruggedness for animal habitat analysis: - a case study using bighorn sheep in the Mojave Desert. The - Journal of wildlife management, 71(5), 1419-1426. - - hgt : array - Array of heights over which TRI will be calculated - window : int - Length of window in x and y direction. Must be odd. - ''' - import richdem as rd - from scipy.ndimage.filters import generic_filter - - # Window setup: - if footprint is not None: - assert window is None, 'Must specify either window or footprint' - window = np.shape(footprint)[0] - - assert (window/2.0) - np.floor(window/2.0) != 0.0, 'window must be odd...' - Hwndw = int(np.floor(window/2)) - - # Type and dimension check: - if isinstance(hgt,(xr.Dataset,xr.DataArray,xr.Variable)): - hgt = hgt.data - assert len(np.shape(hgt)) == 2, 'hgt must be 2-dimensional. Currently has {} dimensions'.format(len(np.shape(hgt))) - ny,nx = np.shape(hgt) - - # Get slope and aspect: - hgt_rd = rd.rdarray(hgt, no_data=-9999) - rd.FillDepressions(hgt_rd, in_place=True) - #slope = rd.TerrainAttribute(hgt_rd, attrib='slope_riserun', zscale=slope_zscale) - zscale = 1/res - slope = rd.TerrainAttribute(hgt_rd, attrib='slope_degrees',zscale=zscale) - aspect = rd.TerrainAttribute(hgt_rd, attrib='aspect') - - # Calculate vectors: - vrm = np.zeros((ny,nx)) - rugz = np.cos(np.deg2rad(slope)) - rugdxy = np.sin(np.deg2rad(slope)) - rugx = rugdxy*np.cos(np.deg2rad(aspect)) - rugy = rugdxy*np.sin(np.deg2rad(aspect)) - - def vrm_filt(x): - return(sum(x)**2) - - if footprint is None: - vrmX = generic_filter(rugx,vrm_filt, size = (window,window)) - vrmY = generic_filter(rugy,vrm_filt, size = (window,window)) - vrmZ = generic_filter(rugz,vrm_filt, size = (window,window)) - else: - vrmX = generic_filter(rugx,vrm_filt, footprint=footprint) - vrmY = generic_filter(rugy,vrm_filt, footprint=footprint) - vrmZ = generic_filter(rugz,vrm_filt, footprint=footprint) - - - if footprint is not None: - num_points = len(footprint[footprint != 0.0]) - else: - num_points = float(window**2) - vrm = 1.0 - np.sqrt(vrmX + vrmY + vrmZ)/num_points - if return_slope: - return vrm,slope - else: - return vrm From 67d098f56affa0edb7ea52449cec2d453d28dbfb Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 12:48:21 -0600 Subject: [PATCH 120/145] Adjusting return to work better with smoothing --- mmctools/wrf/preprocessing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index c2457fd..91b819a 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1728,7 +1728,9 @@ def smooth_eta_levels(self, final_eta_levels -= min(final_eta_levels) final_eta_levels /= max(final_eta_levels) - return(final_eta_levels) + self.original_eta_levels = self.eta_levels + self.eta_levels = final_eta_levels + return(self) def print_eta_levels(self,ncols=4): From 60ef4d0f2b9464c4ac026443987dd7573c97e62a Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 14:42:41 -0600 Subject: [PATCH 121/145] Updating TD for normalized plotting --- mmctools/plotting.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/mmctools/plotting.py b/mmctools/plotting.py index 7caee31..119824c 100644 --- a/mmctools/plotting.py +++ b/mmctools/plotting.py @@ -1886,7 +1886,10 @@ def __init__(self, refstd, corrticks=[0, 0.2, 0.4, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1], minorcorrticks=None, stdevticks=None, - labelsize=None): + labelsize=None, + labelcoraxis=True, + labelstdaxis=True, + extend_length=np.pi): """ Set up Taylor diagram axes, i.e. single quadrant polar plot, using `mpl_toolkits.axisartist.floating_axes`. @@ -1916,6 +1919,10 @@ def __init__(self, refstd, integer input) or FixedLocator (with list-like input) labelsize: int or str, optional Font size (e.g., 16 or 'x-large') for all axes labels + labelaxis: bool, optional + Show axis labels or not (useful for array of plots) + extend_length: float, optional + Extend to a specified radian (default = pi) """ from matplotlib.projections import PolarAxes @@ -1934,7 +1941,7 @@ def __init__(self, refstd, rlocs = np.array(sorted(list(corrticks) + list(minorcorrticks))) if extend: # Diagram extended to negative correlations - self.tmax = np.pi + self.tmax = extend_length rlocs = np.concatenate((-rlocs[:0:-1], rlocs)) else: # Diagram limited to positive correlations @@ -1983,14 +1990,22 @@ def __init__(self, refstd, ax.axis["top"].toggle(ticklabels=True, label=True) ax.axis["top"].major_ticklabels.set_axis_direction("top") ax.axis["top"].label.set_axis_direction("top") - ax.axis["top"].label.set_text("Correlation") + if labelcoraxis: + ax.axis["top"].label.set_text("Correlation") + else: + ax.axis["top"].label.set_text('') + # - "x" axis ax.axis["left"].set_axis_direction("bottom") - if normalize: - ax.axis["left"].label.set_text("Normalized standard deviation") + if labelstdaxis: + if normalize: + left_axis_str = "Normalized standard deviation" + else: + left_axis_str = "Standard deviation" else: - ax.axis["left"].label.set_text("Standard deviation") + left_axis_str = '' + ax.axis["left"].label.set_text(left_axis_str) # - "y" axis ax.axis["right"].set_axis_direction("top") # "Y-axis" From 1c4333dea339e6c24b2094fb9c484babd061b047 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 14:43:24 -0600 Subject: [PATCH 122/145] Small bug fixes --- mmctools/helper_functions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 1085a12..c1fd46e 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -295,7 +295,7 @@ def power_spectral_density(df,tstart=None,interval=None,window_size='10min', # Determine sampling rate and samples per window dts = np.diff(timevalues.unique())/timescale - dt = dts[0] + dt = np.nanmean(dts) if type(window_type) is str: nperseg = int( pd.to_timedelta(window_size)/pd.to_timedelta(dt,'s') ) @@ -1068,9 +1068,8 @@ def get_nc_file_times(f_dir, f_time.append(datetime(time_start.year, time_start.month, time_start.day) + timedelta(seconds=int(nc_time))) else: f_time = ncf[time_dim].data - for ft in f_time: - ft = pd.to_datetime(ft) + ft = pd.to_datetime(str(ft)) file_times[ft] = fname return (file_times) @@ -1248,7 +1247,6 @@ def calc_spectra(data, for varn in list(spec_dat.variables.keys()): if (varn != var_oi) and (varn != spectra_dim): spec_dat = spec_dat.drop(varn) - spec_dat_df = spec_dat[var_oi].to_dataframe() psd = power_spectral_density(spec_dat_df, From 11e38ffb50bd68c1c7102a91fae9bd282dd597df Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 14:44:08 -0600 Subject: [PATCH 123/145] Several changes to WRFSetup script + SST Overwrite --- mmctools/wrf/preprocessing.py | 286 ++++++++++++++++++++++++---------- 1 file changed, 208 insertions(+), 78 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index ad7d63f..9fbc72a 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -429,7 +429,7 @@ def _get_icbc_info(self): interval_seconds = 3600 met_lvls = 38 download_freq = '1h' - elif icbc_type == 'FNL': + elif 'FNL' in icbc_type: met_lvls = 34 elif icbc_type == 'NARR': met_lvls = 30 @@ -459,8 +459,10 @@ def _check_namelist_opts(self): missing_options = True if not missing_options: - if 'usgs' in self.setup_dict['geogrid_args']: + if 'usgs+' in self.setup_dict['geogrid_args']: land_cat = 24 + elif 'usgs_lakes+' in self.setup_dict['geogrid_args']: + land_cat = 28 else: land_cat = 21 namelist_defaults = { @@ -501,6 +503,7 @@ def _check_namelist_opts(self): 'dampcoef' : 0.2, 'khdif' : 0, 'kvdif' : 0, + 'smdiv' : 0.1, 'non_hydrostatic' : '.true.', 'moist_adv_opt' : 1, 'scalar_adv_opt' : 1, @@ -509,12 +512,15 @@ def _check_namelist_opts(self): 'v_mom_adv_order' : 3, 'h_sca_adv_order' : 5, 'v_sca_adv_order' : 3, + 'gwd_opt' : 0, 'spec_bdy_width' : 5, 'spec_zone' : 1, 'relax_zone' : 4, 'nio_tasks_per_group' : 0, 'nio_groups' : 1, 'sst_update' : 1, + 'sst_skin' : 0, + 'sf_ocean_physics' : 0, } namelist_opts = namelist_defaults @@ -654,15 +660,17 @@ def write_namelist_input(self): end_second_str = "{0:>5},".format(end_date.second)*num_doms parent_ids,grid_ids,dx_str,radt_str = '','','','' + full_pgr = self.namelist_opts['parent_grid_ratio'].copy() for pp,pgr in enumerate(self.namelist_opts['parent_grid_ratio']): + full_pgr[pp] = np.prod(self.namelist_opts['parent_grid_ratio'][:pp+1]) grid_ids += '{0:>5},'.format(str(pp+1)) if pp == 0: pid = 1 else: pid = pp parent_ids += '{0:>5},'.format(str(pid)) - dx_str += '{0:>5},'.format(str(int(self.namelist_opts['dxy']/np.prod(self.namelist_opts['parent_grid_ratio'][:(pp+1)])))) - radt = self.namelist_opts['radt']/pgr + dx_str += '{0:>5},'.format(str(int(self.namelist_opts['dxy']/full_pgr[pp]))) + radt = self.namelist_opts['radt']/full_pgr[pp] if radt < 1: radt = 1 radt_str += '{0:>5},'.format(str(int(radt))) @@ -724,6 +732,7 @@ def write_namelist_input(self): damp_str = self._get_nl_str(num_doms,self.namelist_opts['dampcoef']) khdif_str = self._get_nl_str(num_doms,self.namelist_opts['khdif']) kvdif_str = self._get_nl_str(num_doms,self.namelist_opts['kvdif']) + smdiv_str = self._get_nl_str(num_doms,self.namelist_opts['smdiv']) nonhyd_str = self._get_nl_str(num_doms,self.namelist_opts['non_hydrostatic']) moist_str = self._get_nl_str(num_doms,self.namelist_opts['moist_adv_opt']) scalar_str = self._get_nl_str(num_doms,self.namelist_opts['scalar_adv_opt']) @@ -732,6 +741,9 @@ def write_namelist_input(self): vmom_str = self._get_nl_str(num_doms,self.namelist_opts['v_mom_adv_order']) hsca_str = self._get_nl_str(num_doms,self.namelist_opts['h_sca_adv_order']) vsca_str = self._get_nl_str(num_doms,self.namelist_opts['v_sca_adv_order']) + gwd_str = self._get_nl_str(num_doms,self.namelist_opts['gwd_opt']) + if 'shalwater_rough' in self.namelist_opts: + shalwater_rough_str = self._get_nl_str(num_doms,self.namelist_opts['shalwater_rough']) specified = ['.false.']*num_doms nested = ['.true.']*num_doms @@ -795,6 +807,10 @@ def write_namelist_input(self): f.write(" e_vert = {}\n".format("{0:>5},".format(self.namelist_opts['num_eta_levels'])*num_doms)) if 'eta_levels' in self.namelist_opts.keys(): f.write(" eta_levels = {},\n".format(self.namelist_opts['eta_levels'])) + if 'dzbot' in self.namelist_opts.keys(): + f.write(" dzbot = {},\n".format(self.namelist_opts['dzbot'])) + if 'dzstretch_s' in self.namelist_opts.keys(): + f.write(" dzstretch_s = {},\n".format(self.namelist_opts['dzstretch_s'])) f.write(" p_top_requested = {},\n".format(self.namelist_opts['p_top_requested'])) f.write(" num_metgrid_levels = {},\n".format(self.namelist_opts['num_metgrid_levels'])) f.write(" num_metgrid_soil_levels = {},\n".format(self.namelist_opts['num_metgrid_soil_levels'])) @@ -808,6 +824,9 @@ def write_namelist_input(self): f.write(" parent_time_step_ratio = {}\n".format(parent_time_ratios)) f.write(" feedback = {},\n".format(self.namelist_opts['feedback'])) f.write(" smooth_option = {},\n".format(self.namelist_opts['smooth_option'])) + if ('nproc_x' in self.namelist_opts.keys()) and ('nproc_y' in self.namelist_opts.keys()): + f.write(" nproc_x = {},\n".format(self.namelist_opts['nproc_x'])) + f.write(" nproc_y = {},\n".format(self.namelist_opts['nproc_y'])) f.write(" /\n") f.write("\n") f.write("&physics\n") @@ -829,10 +848,40 @@ def write_namelist_input(self): f.write(" num_land_cat = {}, \n".format(self.namelist_opts['num_land_cat'])) f.write(" sf_urban_physics = {}\n".format(urb_str)) f.write(" sst_update = {}, \n".format(self.namelist_opts['sst_update'])) + f.write(" sst_skin = {}, \n".format(self.namelist_opts['sst_skin'])) + f.write(" sf_ocean_physics = {}, \n".format(self.namelist_opts['sf_ocean_physics'])) + + if 'shalwater_rough' in self.namelist_opts: + f.write(" shalwater_rough = {} \n".format(shalwater_rough_str)) + if 'shalwater_depth' in self.namelist_opts: + f.write(" shalwater_depth = {}, \n".format(self.namelist_opts['shalwater_depth'])) f.write(" /\n") + f.write("\n") f.write("&fdda\n") + if 'fdda_dict' in self.namelist_opts: + f.write("grid_fdda = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['grid_fdda']))) + f.write('gfdda_inname = "{}",\n'.format(self.namelist_opts['fdda_dict']['gfdda_inname'])) + f.write("gfdda_interval_m = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['gfdda_interval_m']))) + f.write("io_form_gfdda = {},\n".format(self.namelist_opts['fdda_dict']['io_form_gfdda'])) + f.write("if_no_pbl_nudging_uv = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['if_no_pbl_nudging_uv']))) + f.write("if_no_pbl_nudging_t = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['if_no_pbl_nudging_t']))) + f.write("if_no_pbl_nudging_ph = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['if_no_pbl_nudging_ph']))) + f.write("if_zfac_uv = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['if_zfac_uv']))) + f.write("k_zfac_uv = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['k_zfac_uv']))) + f.write("if_zfac_t = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['if_zfac_t']))) + f.write("k_zfac_t = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['k_zfac_t']))) + f.write("if_zfac_ph = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['if_zfac_ph']))) + f.write("k_zfac_ph = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['k_zfac_ph']))) + f.write("guv = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['guv']))) + f.write("gt = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['gt']))) + f.write("gph = {}\n".format(self._get_nl_str(num_doms,self.namelist_opts['fdda_dict']['gph']))) + f.write("if_ramping = {},\n".format(self.namelist_opts['fdda_dict']['if_ramping'])) + f.write("dtramp_min = {},\n".format(self.namelist_opts['fdda_dict']['dtramp_min'])) + f.write("xwavenum = {},\n".format(self.namelist_opts['fdda_dict']['xwavenum'])) + f.write("ywavenum = {},\n".format(self.namelist_opts['fdda_dict']['ywavenum'])) f.write("/\n") + f.write("\n") f.write("&dynamics\n") if 'hybrid_opt' in self.namelist_opts: @@ -851,6 +900,8 @@ def write_namelist_input(self): f.write(" dampcoef = {}\n".format(damp_str)) f.write(" khdif = {}\n".format(khdif_str)) f.write(" kvdif = {}\n".format(kvdif_str)) + if 'smdiv' in self.namelist_opts: + f.write(" smdiv = {}\n".format(smdiv_str)) f.write(" non_hydrostatic = {}\n".format(nonhyd_str)) f.write(" moist_adv_opt = {}\n".format(moist_str)) f.write(" scalar_adv_opt = {}\n".format(scalar_str)) @@ -859,6 +910,7 @@ def write_namelist_input(self): f.write(" v_mom_adv_order = {}\n".format(vmom_str)) f.write(" h_sca_adv_order = {}\n".format(hsca_str)) f.write(" v_sca_adv_order = {}\n".format(vsca_str)) + f.write(" gwd_opt = {}\n".format(gwd_str)) f.write(" /\n") f.write(" \n") f.write("&bdy_control\n") @@ -886,7 +938,7 @@ def get_icbcs(self): icbc_type = self.namelist_opts['icbc_type'].upper() if icbc_type == 'ERAI': icbc = ERAInterim() - elif icbc_type == 'FNL': + elif 'FNL' in icbc_type: icbc = FNL() elif icbc_type == 'MERRA2': print('Cannot download MERRA2 yet... please download manually and put in the IC/BC dir:') @@ -919,8 +971,11 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): if hpc == 'cheyenne': f = open('{}submit_{}.sh'.format(self.run_dir,executable),'w') f.write("#!/bin/bash\n") + case_str = self.run_dir.split('/')[-3].split('_')[0] run_str = '{0}{1}'.format(self.icbc_dict['type'], (self.setup_dict['start_date'].split(' ')[0]).replace('-','')) + run_str = '{0}{1}'.format(case_str, + (self.setup_dict['start_date'].split(' ')[0]).replace('-','')) f.write("#PBS -N {} \n".format(run_str)) f.write("#PBS -A {}\n".format(submission_dict['account_key'])) f.write("#PBS -l walltime={0:02d}:00:00\n".format(submission_dict['walltime_hours'][executable])) @@ -930,9 +985,9 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): f.write("#PBS -M {}\n".format(submission_dict['user_email'])) f.write("### Select 2 nodes with 36 CPUs each for a total of 72 MPI processes\n") if executable == 'wps': - f.write("#PBS -l select=1:ncpus=1:mpiprocs=1\n".format(submission_dict['nodes'])) + f.write("#PBS -l select=1:ncpus=1:mpiprocs=1\n") else: - f.write("#PBS -l select={0:02d}:ncpus=36:mpiprocs=36\n".format(submission_dict['nodes'])) + f.write("#PBS -l select={0:02d}:ncpus=36:mpiprocs=36\n".format(submission_dict['nodes'][executable])) f.write("date_start=`date`\n") f.write("echo $date_start\n") f.write("module list\n") @@ -944,7 +999,7 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): elif icbc_type == 'ERAI': icbc_head = 'ei.oper*' icbc_vtable = 'ERA-interim.pl' - elif icbc_type == 'FNL': + elif 'FNL' in icbc_type: icbc_head = 'fnl_*' icbc_vtable = 'GFS' elif icbc_type == 'MERRA2': @@ -1051,6 +1106,11 @@ def create_submitAll_scripts(self,main_directory,list_of_cases,executables): def create_tslist_file(self,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): fname = '{}tslist'.format(self.run_dir) write_tslist_file(fname,lat=lat,lon=lon,i=i,j=j,twr_names=twr_names,twr_abbr=twr_abbr) + + def link_metem_files(self,met_em_dir): + # Link WPS and WRF files / executables + met_files = glob.glob('{}/*'.format(met_em_dir)) + self._link_files(met_files,self.run_dir) def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): @@ -1148,6 +1208,46 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a 'sst_dx' : 5.5, # km }, + 'NAVO' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 10.0, # km + }, + + 'OSPO' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 5.5, # km + }, + + 'NCEI' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 27.75, # km + }, + + 'CMC' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 11.1, # km + }, + + 'G1SST' : { + 'time_dim' : 'time', + 'lat_dim' : 'lat', + 'lon_dim' : 'lon', + 'sst_name' : 'analysed_sst', + 'sst_dx' : 1.0, # km + }, + 'MUR' : { 'time_dim' : 'time', 'lat_dim' : 'lat', @@ -1185,6 +1285,10 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a 'ERA5' : { 'sst_name' : 'SST', }, + + 'MERRA2' : { + 'sst_name' : 'SKINTEMP', + }, } @@ -1206,7 +1310,8 @@ class OverwriteSST(): fill_missing = boolean; fill missing values in SST data with SKINTEMP ''' - + import warnings + warnings.filterwarnings("ignore", category=RuntimeWarning) def __init__(self, met_type, overwrite_type, @@ -1214,7 +1319,8 @@ def __init__(self, sst_directory, out_directory, smooth_opt=False, - fill_missing=False): + fill_missing=False, + skip_finished=True): self.met_type = met_type self.overwrite = overwrite_type @@ -1223,6 +1329,7 @@ def __init__(self, self.out_dir = out_directory self.smooth_opt = smooth_opt self.fill_opt = fill_missing + self.skip_finished = skip_finished if overwrite_type == 'FILL': fill_missing=True @@ -1230,27 +1337,45 @@ def __init__(self, self.smooth_str = 'smooth' else: self.smooth_str = 'raw' - - self.out_dir += '{}/'.format(self.smooth_str) + if (self.smooth_str == 'raw') and fill_missing: + self.smooth_str += '-filled' + + self.out_dir += '{}/'.format(self.smooth_str) + if not os.path.exists(self.out_dir): + os.mkdir(self.out_dir) + # Get met_em_files self.met_em_files = sorted(glob.glob('{}met_em.d0*'.format(self.met_dir))) + print(self.met_em_files) + # Get SST data info (if not doing fill or tskin) if (overwrite_type.upper() != 'FILL') and (overwrite_type.upper() != 'TSKIN'): self._get_sst_info() # Overwrite the met_em SST data: - for mm,met_file in enumerate(self.met_em_files): - self._get_new_sst(met_file) - # If filling missing values with SKINTEMP: - if fill_missing: - self._fill_missing(met_file) - self.new_sst = np.nan_to_num(self.new_sst) - # Write to new file: - self._write_new_file(met_file) + for mm,met_file in enumerate(self.met_em_files[:]): + self._check_file_exists(met_file) + if self.exists and self.skip_finished: + print('{} exists... skipping'.format(self.new_file)) + else: + print(self.new_file) + self._get_new_sst(met_file) + # If filling missing values with SKINTEMP: + if fill_missing: + self._fill_missing(met_file) + self.new_sst = np.nan_to_num(self.new_sst) + # Write to new file: + self._write_new_file(met_file) + def _check_file_exists(self,met_file): + f_name = met_file.split('/')[-1] + self.new_file = self.out_dir + f_name + self.exists = os.path.exists(self.new_file) + + def _get_sst_info(self): if self.overwrite == 'MODIS': @@ -1290,14 +1415,20 @@ def _get_sst_info(self): self.sst_file_times = sst_file_times sst = xr.open_dataset(sst_file_times[list(sst_file_times.keys())[0]]) - self.sst_lat = sst[sst_dict[self.overwrite]['lat_dim']] - self.sst_lon = sst[sst_dict[self.overwrite]['lon_dim']] + sst_lat = sst[sst_dict[self.overwrite]['lat_dim']] + sst_lon = sst[sst_dict[self.overwrite]['lon_dim']] + + self.sst_lat = sst_lat + self.sst_lon = sst_lon + + def _get_new_sst(self,met_file): - met = xr.open_dataset(met_file) + import matplotlib.pyplot as plt + met = xr.open_dataset(met_file) met_time = pd.to_datetime(met.Times.data[0].decode().replace('_',' ')) met_lat = np.squeeze(met.XLAT_M) met_lon = np.squeeze(met.XLONG_M) @@ -1322,16 +1453,18 @@ def _get_new_sst(self,met_file): before_ds = xr.open_dataset(self.sst_file_times[sst_neighbors[0]]) after_ds = xr.open_dataset(self.sst_file_times[sst_neighbors[1]]) - min_lon = np.max([np.nanmin(met_lon)-1,-180]) - max_lon = np.min([np.nanmax(met_lon)+1,180]) - if (self.overwrite == 'MODIS') or (self.overwrite == 'GOES16'): min_lat = np.min([np.nanmax(met_lat)+1,90]) max_lat = np.max([np.nanmin(met_lat)-1,-90]) else: + before_ds = before_ds.sortby('lat') + after_ds = after_ds.sortby('lat') min_lat = np.max([np.nanmin(met_lat)-1,-90]) max_lat = np.min([np.nanmax(met_lat)+1,90]) + min_lon = np.max([np.nanmin(met_lon)-1,-180]) + max_lon = np.min([np.nanmax(met_lon)+1,180]) + before_ds = before_ds.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) after_ds = after_ds.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), @@ -1354,53 +1487,56 @@ def _get_new_sst(self,met_file): sst_lat = self.sst_lat.data sst_lon = self.sst_lon.data - + for jj in met.south_north: for ii in met.west_east: if met_landmask[jj,ii] == 0.0: - dist_lat = abs(sst_lat - float(met_lat[jj,ii])) - dist_lon = abs(sst_lon - float(met_lon[jj,ii])) - - lat_ind = np.where(dist_lat==np.min(dist_lat))[0] - lon_ind = np.where(dist_lon==np.min(dist_lon))[0] - - if (len(lat_ind) > 1) and (len(lon_ind) > 1): - lat_s = sst_lat[lat_ind[0] - window] - lat_e = sst_lat[lat_ind[1] + window] - lon_s = sst_lon[lon_ind[0] - window] - lon_e = sst_lon[lon_ind[1] + window] - - elif (len(lat_ind) > 1) and (len(lon_ind) == 1): - lat_s = sst_lat[lat_ind[0] - window] - lat_e = sst_lat[lat_ind[1] + window] - lon_s = sst_lon[lon_ind[0] - window] - lon_e = sst_lon[lon_ind[0] + window] - - elif (len(lat_ind) == 1) and (len(lon_ind) > 1): - lat_s = sst_lat[lat_ind[0] - window] - lat_e = sst_lat[lat_ind[0] + window] - lon_s = sst_lon[lon_ind[0] - window] - lon_e = sst_lon[lon_ind[1] + window] - - else: - lat_s = sst_lat[lat_ind[0] - window] - lat_e = sst_lat[lat_ind[0] + window] - lon_s = sst_lon[lon_ind[0] - window] - try: + within_lat = (np.nanmin(sst_lat) <= met_lat[jj,ii] <= np.nanmax(sst_lat)) + within_lon = (np.nanmin(sst_lon) <= met_lon[jj,ii] <= np.nanmax(sst_lon)) + if within_lat and within_lon: + dist_lat = abs(sst_lat - float(met_lat[jj,ii])) + dist_lon = abs(sst_lon - float(met_lon[jj,ii])) + + lat_ind = np.where(dist_lat==np.min(dist_lat))[0] + lon_ind = np.where(dist_lon==np.min(dist_lon))[0] + + if (len(lat_ind) > 1) and (len(lon_ind) > 1): + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[1] + window] + lon_s = sst_lon[lon_ind[0] - window] + lon_e = sst_lon[lon_ind[1] + window] + + elif (len(lat_ind) > 1) and (len(lon_ind) == 1): + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[1] + window] + lon_s = sst_lon[lon_ind[0] - window] lon_e = sst_lon[lon_ind[0] + window] - except IndexError: - lon_e = len(sst_lon) - - sst_before_val = before_sst.sel({ - sst_dict[self.overwrite]['lat_dim']:slice(lat_s,lat_e), - sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) - - sst_after_val = after_sst.sel({ - sst_dict[self.overwrite]['lat_dim']:slice(lat_s,lat_e), - sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) - new_sst[jj,ii] = sst_before_val*sst_weights[0] + sst_after_val*sst_weights[1] - + elif (len(lat_ind) == 1) and (len(lon_ind) > 1): + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[0] + window] + lon_s = sst_lon[lon_ind[0] - window] + lon_e = sst_lon[lon_ind[1] + window] + + else: + lat_s = sst_lat[lat_ind[0] - window] + lat_e = sst_lat[lat_ind[0] + window] + lon_s = sst_lon[lon_ind[0] - window] + try: + lon_e = sst_lon[lon_ind[0] + window] + except IndexError: + lon_e = len(sst_lon) + + sst_before_val = before_sst.sel({ + sst_dict[self.overwrite]['lat_dim']:slice(lat_s,lat_e), + sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) + + sst_after_val = after_sst.sel({ + sst_dict[self.overwrite]['lat_dim']:slice(lat_s,lat_e), + sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) + + new_sst[jj,ii] = sst_before_val*sst_weights[0] + sst_after_val*sst_weights[1] + else: if (self.overwrite.upper() == 'TSKIN'): new_sst = met_sst.data.copy() @@ -1408,8 +1544,8 @@ def _get_new_sst(self,met_file): new_sst[np.where(met_landmask==0.0)] = tsk[np.where(met_landmask==0.0)] elif (self.overwrite.upper() == 'FILL'): new_sst = np.squeeze(met[icbc_dict[self.met_type]['sst_name']]).data + self.new_sst = new_sst - def _get_closest_files(self,met_time): sst_times = np.asarray(list(self.sst_file_times.keys())) @@ -1423,7 +1559,6 @@ def _get_closest_files(self,met_time): time_dist[dt] = abs(stime - met_time) closest_time = sst_times[np.where(time_dist == np.min(time_dist))] - if len(closest_time) == 1: if closest_time == met_time: sst_before = closest_time[0] @@ -1440,10 +1575,10 @@ def _get_closest_files(self,met_time): got_before = True else: sst_after = closest_time + if closest_ind <= 1: closest_ind += 1 next_closest_times = sst_times[:closest_ind-1] next_closest_dist = time_dist[:closest_ind-1] got_after = True - next_closest_time = next_closest_times[np.where(next_closest_dist == np.min(next_closest_dist))][0] if got_before: @@ -1487,14 +1622,9 @@ def _fill_missing(self,met_file): def _write_new_file(self,met_file): f_name = met_file.split('/')[-1] - new_file = self.out_dir + f_name - print(new_file) new = xr.open_dataset(met_file) - new.SST.data = np.expand_dims(self.new_sst,axis=0) + new[icbc_dict[self.met_type]['sst_name']].data = np.expand_dims(self.new_sst,axis=0) new.attrs['source'] = '{}'.format(self.overwrite) new.attrs['smoothed'] = '{}'.format(self.smooth_opt) new.attrs['filled'] = '{}'.format(self.fill_opt) - if os.path.exists(new_file): - print('File exists... replacing') - os.remove(new_file) - new.to_netcdf(new_file) + new.to_netcdf(self.new_file) From df033f2351827dfee3e9bc660e6a4e7181eeab78 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 16:41:43 -0600 Subject: [PATCH 124/145] Adjusting eta level calculation based on comments Eliot and I spoke and made the code more readable and eliminated redundancies / unused variables. I then added an estimator for the resulting heights that would come out of WRF. --- mmctools/wrf/preprocessing.py | 82 ++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 91b819a..86e7f9c 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1593,8 +1593,7 @@ class CreateEtaLevels(): pres_top=62500.0, surface_temp=290.0, height_top=4000, - n_total_levels=201, - smooth_eta=False + n_total_levels=201 ) @@ -1604,8 +1603,7 @@ class CreateEtaLevels(): pres_top=10000.0, surface_temp=282.72, height_top=16229.028, - n_total_levels=88, - smooth_eta=False + n_total_levels=88 ) 3. Smooth the eta levels so there are no harsh jumps in d(eta): @@ -1614,21 +1612,29 @@ class CreateEtaLevels(): pres_top=10000.0, surface_temp=282.72, height_top=16229.028, - n_total_levels=88, - smooth_eta=True + n_total_levels=88 ) ''' + gas_constant_dry_air = 287.06 + gravity = 9.80665 + M = 0.0289644 + universal_gas_constant = 8.3144598 + def __init__(self, levels=None, eta_levels=None, surface_temp=290, pres_top=None, - height_top=14417.41, + height_top=None, p0=100000.0, fill_to_top=True, n_total_levels=None): + self.p0 = p0 + self.pres_top = pres_top + self.surface_temp = surface_temp + if eta_levels is not None: self.eta_levels = eta_levels else: @@ -1644,53 +1650,50 @@ def __init__(self, levels=None, self.levels = levels - if n_total_levels < len(self.levels): - print('Setting n_total_levels to be len(levels).') + if n_total_levels is not None: + if n_total_levels < len(self.levels): + print('Setting n_total_levels to be len(levels).') + n_total_levels = len(levels) + else: n_total_levels = len(levels) - pressure = self._pressure_calc(surface_temp,height_top,p0) + pressure = self._pressure_calc() + if pres_top is None: - pres_top = pressure[-1] + self.pres_top = pressure[-1] + - self.eta_levels = self._eta_level_calc(pressure,pres_top,height_top,p0,n_total_levels,fill_to_top) + self.eta_levels = self._eta_level_calc(pressure, + height_top, + n_total_levels, + fill_to_top) + self.estimated_heights = self._estimate_heights() + + def _pressure_calc(self): - def _pressure_calc(self,surface_temp, - height_top, - p0, - pres_calc_option=1): - gas_constant_dry_air = 287.06 - gravity = 9.80665 - M = 0.0289644 - universal_gas_constant = 8.3144598 - - if pres_calc_option == 1: - pressure = p0*np.exp((-gravity*self.levels)/gas_constant_dry_air/surface_temp) - elif pres_calc_option == 2: - pressure = p0*np.exp((-gravity*M*self.levels/(universal_gas_constant*surface_temp))) - else: - print('pres_calc_option = {} is not a valid option. Please select 1 or 2'.format(pres_calc_option)) - - return(pressure) + return(self.p0*np.exp((-self.gravity*self.levels)/self.gas_constant_dry_air/self.surface_temp)) - def _eta_level_calc(self,pressure, - pres_top, + def _eta_level_calc(self, + pressure, height_top, - p0, n_total_levels, fill_to_top): + if height_top is None: + height_top = (self.gas_constant_dry_air*self.surface_temp/self.gravity)*np.log((self.p0/self.pres_top)) + eta_levels = np.zeros(n_total_levels) - eta_levels[:len(self.levels)] = (pressure-pres_top)/(p0-pres_top) + eta_levels[:len(self.levels)] = (pressure-self.pres_top)/(self.p0-self.pres_top) - if np.max(self.levels) < height_top: + if float(np.max(self.levels)) < float(height_top): if (n_total_levels is None) or (n_total_levels <= len(self.levels)): print('Insufficient number of levels to reach model top.') print('Height top: {}, top of specified levels: {}, number of levels: {}'.format( height_top,self.levels[-1],n_total_levels)) - print('Must specify n_total_levels to complete eta_levels to model top') - return + raise ValueError ('Must specify n_total_levels to complete eta_levels to model top') + remaining_levels = n_total_levels - len(self.levels) eta_levels_top = np.zeros(remaining_levels+2) @@ -1707,6 +1710,12 @@ def _eta_level_calc(self,pressure, return(eta_levels) + def _estimate_heights(self): + + pressure = self.eta_levels*(self.p0-self.pres_top) + self.pres_top + return((self.gas_constant_dry_air*self.surface_temp/self.gravity)*np.log((self.p0/pressure))) + + def smooth_eta_levels(self, smooth_fact=7e-4, smooth_degree=2): @@ -1730,6 +1739,7 @@ def smooth_eta_levels(self, self.original_eta_levels = self.eta_levels self.eta_levels = final_eta_levels + self.estimated_heights = self._estimate_heights() return(self) From 237a986d175e3216b30e219090881a42ccb9a7b4 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 16:47:01 -0600 Subject: [PATCH 125/145] Cleanup docstrings --- mmctools/wrf/preprocessing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 86e7f9c..f614b64 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1599,7 +1599,7 @@ class CreateEtaLevels(): 2. Only specify lower levels and let the program fill the rest (no smoothing): - eta_levels = generate_eta_levels(levels=np.linspace(0,1000,50), + eta_levels = generate_eta_levels(levels=np.linspace(0,1000,51), pres_top=10000.0, surface_temp=282.72, height_top=16229.028, @@ -1613,7 +1613,7 @@ class CreateEtaLevels(): surface_temp=282.72, height_top=16229.028, n_total_levels=88 - ) + ).smooth_eta_levels(smooth_fact=9e-4) ''' From ab0f8ec5e8fcc3ae7d0312519887e048abcc33dd Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Tue, 28 Sep 2021 16:50:10 -0600 Subject: [PATCH 126/145] Bug fix for specifying eta levels When eta levels are given (not calculated) then the estimation of model heights cannot be performed without specifying surface temp, pressure, etc. Added a check for this. --- mmctools/wrf/preprocessing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index f614b64..4a9ecbf 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1624,7 +1624,7 @@ class CreateEtaLevels(): def __init__(self, levels=None, eta_levels=None, - surface_temp=290, + surface_temp=None, pres_top=None, height_top=None, p0=100000.0, @@ -1739,7 +1739,8 @@ def smooth_eta_levels(self, self.original_eta_levels = self.eta_levels self.eta_levels = final_eta_levels - self.estimated_heights = self._estimate_heights() + if self.pres_top is not None: + self.estimated_heights = self._estimate_heights() return(self) From 3bdae572ebf79ef1e534b548187ef9d215224323 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Wed, 29 Sep 2021 18:07:53 -0600 Subject: [PATCH 127/145] Restructuring eta level code Last version wasn't good. This one is constructing from d(eta) levels and produces a much smoother set. Will test if it works. --- mmctools/wrf/preprocessing.py | 175 ++++++++++++++++++++++++++++++---- 1 file changed, 154 insertions(+), 21 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index 4a9ecbf..db4717b 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -1620,7 +1620,9 @@ class CreateEtaLevels(): gas_constant_dry_air = 287.06 gravity = 9.80665 M = 0.0289644 - universal_gas_constant = 8.3144598 + universal_gas_constant = 8.3144598 + + deta_limit = -0.035 def __init__(self, levels=None, eta_levels=None, @@ -1629,7 +1631,9 @@ def __init__(self, levels=None, height_top=None, p0=100000.0, fill_to_top=True, - n_total_levels=None): + n_total_levels=None, + transition_zone=None, + min_transition_deta=None): self.p0 = p0 self.pres_top = pres_top @@ -1654,6 +1658,8 @@ def __init__(self, levels=None, if n_total_levels < len(self.levels): print('Setting n_total_levels to be len(levels).') n_total_levels = len(levels) + elif transition_zone is not None: + n_total_levels = len(levels) + transition_zone else: n_total_levels = len(levels) @@ -1666,7 +1672,9 @@ def __init__(self, levels=None, self.eta_levels = self._eta_level_calc(pressure, height_top, n_total_levels, - fill_to_top) + fill_to_top, + transition_zone, + min_transition_deta) self.estimated_heights = self._estimate_heights() @@ -1678,38 +1686,163 @@ def _eta_level_calc(self, pressure, height_top, n_total_levels, - fill_to_top): + fill_to_top, + transition_zone, + min_transition_deta): if height_top is None: height_top = (self.gas_constant_dry_air*self.surface_temp/self.gravity)*np.log((self.p0/self.pres_top)) - eta_levels = np.zeros(n_total_levels) - - eta_levels[:len(self.levels)] = (pressure-self.pres_top)/(self.p0-self.pres_top) + eta_levels = (pressure-self.pres_top)/(self.p0-self.pres_top) + reached_model_top = False if float(np.max(self.levels)) < float(height_top): + if transition_zone is not None: + if (len(self.levels)+transition_zone) > n_total_levels: + print('Transition zone makes this larger than n_total_levels') + print('Setting n_total_levels to len(levels) + transition_zone') + n_total_levels = len(self.levels)+transition_zone + + transition = np.zeros(transition_zone) + for tt,tran in enumerate(range(1,transition_zone+1)): + overlaying_slope = 0.002 # slope applied to cos curve + cos_tail = 0.3 # How much of the cos curve should continue (0.3 = 30%) + cos_squeeze = 0.9 # How much of the cos curve should be squeezed (1 = none, 0.9 = a little toward the beginning) + transition[tt] = (1.0 + np.cos((tran*((tran/((1-cos_tail)*transition_zone))**cos_squeeze)/transition_zone)*np.pi)) - tt*overlaying_slope + + transition -= np.min(transition) + transition /= np.max(transition) + max_transition_deta = eta_levels[len(self.levels)-1] - eta_levels[len(self.levels)-2] + + + orig_eta_levels = eta_levels + orig_transition = transition.copy() + + top_lvl_threshold = 0.001 + if min_transition_deta is not None: + + eta_levels,error = self._calc_transition(eta_levels, + transition, + max_transition_deta, + min_transition_deta) + + if error > top_lvl_threshold: + iterate_for_transition = True + print('Specified min_transition_deta resulted in bad levels...') + eta_levels = orig_eta_levels.copy() + else: + iterate_for_transition = False + else: + iterate_for_transition = True + min_transition_deta = -0.02 + + if iterate_for_transition: + print('Iterating to find reasonable levels.') + error = 1.0 + count = 0 + max_iterations = 100 + while ((np.abs(error) > top_lvl_threshold) and (count <= max_iterations)) and (min_transition_deta > self.deta_limit): + eta_levels = orig_eta_levels.copy() + transition = orig_transition.copy() + + eta_levels,error = self._calc_transition(eta_levels, + transition, + max_transition_deta, + min_transition_deta, + top_lvl_threshold) + if error > 0.0: + min_transition_deta -= 0.001 + else: + min_transition_deta += 0.0001 + count+=1 + if min_transition_deta < self.deta_limit: + print('min_transition_deta of {} exceeds reasonable limit for d(eta), {}'.format( + min_transition_deta, self.deta_limit)) + #raise ValueError ('Could not find a reasonable min_transition_deta - try adding more levels') + min_transition_deta = self.deta_limit + count = max_iterations + 1 + + + if (count > max_iterations) and len(eta_levels) >= n_total_levels: + raise ValueError ('Not enough levels to reach the top') + else: + if error <= top_lvl_threshold: + eta_levels -= np.min(eta_levels) + eta_levels /= np.max(eta_levels) + + + if np.min(eta_levels) < 0.0: + eta_levels -= np.min(eta_levels) + eta_levels /= np.max(eta_levels) + reached_model_top = True + if (n_total_levels is None) or (n_total_levels <= len(self.levels)): print('Insufficient number of levels to reach model top.') print('Height top: {}, top of specified levels: {}, number of levels: {}'.format( height_top,self.levels[-1],n_total_levels)) raise ValueError ('Must specify n_total_levels to complete eta_levels to model top') - remaining_levels = n_total_levels - len(self.levels) - - eta_levels_top = np.zeros(remaining_levels+2) - z_scale = 0.4 - for k in range(1,remaining_levels+2): - kind = k-1 - eta_levels_top[kind] = (np.exp(-(k-1)/float(n_total_levels)/z_scale) - np.exp(-1./z_scale))/ (1.-np.exp(-1./z_scale)) - eta_levels_top -= eta_levels_top[-2] - eta_levels_top = eta_levels_top[:-1] - eta_levels_top /= np.max(eta_levels_top) - eta_levels_top *= eta_levels[len(self.levels)-1] - - eta_levels[len(self.levels):] = eta_levels_top[1:] - + remaining_levels = n_total_levels - len(eta_levels) + + if remaining_levels > 0: + if not reached_model_top: + print('Filling to top...') + import matplotlib.pyplot as plt + plt.plot(eta_levels) + plt.show() + plt.plot(eta_levels[1:]-eta_levels[:-1]) + plt.show() + eta_levels_top = np.zeros(remaining_levels+2) + z_scale = 0.4 + for k in range(1,remaining_levels+2): + kind = k - 1 + eta_levels_top[kind] = (np.exp(-(k-1)/float(n_total_levels)/z_scale) - np.exp(-1./z_scale))/ (1.-np.exp(-1./z_scale)) + #eta_levels_top[kind] = (np.exp(-(k-1)/float(remaining_levels)/z_scale) - np.exp(-1./z_scale))/ (1.-np.exp(-1./z_scale)) + + eta_levels_top = eta_levels_top[:-1] + print(eta_levels_top) + + eta_levels_top -= np.min(eta_levels_top) + eta_levels_top /= np.max(eta_levels_top) + + eta_levels_top *= eta_levels[-1] + print(eta_levels_top) + + eta_levels_top = list(eta_levels_top) + eta_levels = list(eta_levels) + + eta_levels += eta_levels_top[1:] + eta_levels = np.array(eta_levels) + else: + print('Specified levels + transition zone reached model top.') + print('Setting n_total_levels to len(levels) + transition_zone') + n_total_levels = len(self.levels)+transition_zone + + + if np.min(eta_levels) != 0.0: + print('Insufficient number of levels to reach model top.') + raise ValueError ('Lower the model top, increase number of levels, or increase the deta_lim') + return(eta_levels) + def _calc_transition(self,eta_levels, + transition, + max_transition_deta, + min_transition_deta, + top_lvl_threshold=None): + + transition *= (max_transition_deta - min_transition_deta) + transition += min_transition_deta + + eta_levels = list(eta_levels) + for tt,tran in enumerate(transition): + tind = len(self.levels) + tt + eta_levels += list([eta_levels[tind-1] + tran]) + eta_levels = np.array(eta_levels) + + error = eta_levels[-1] + return (eta_levels,error) + def _estimate_heights(self): pressure = self.eta_levels*(self.p0-self.pres_top) + self.pres_top From ccdeeb84d9680d6557b8bbe3e39f1e46af819716 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 2 Nov 2021 14:44:02 -0600 Subject: [PATCH 128/145] Improved shelterness computation Prior lazy implementation for testing was too slow for practical purposes. This current implementation is significantly faster, as it avoids a `griddata` call for every point. It should still take about 5 minutes to run for a grid with about 300x300 points. --- mmctools/coupling/terrain.py | 76 +++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index f2aebfa..e4bf356 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -625,7 +625,81 @@ def vrm_filt(x): return vrm -def calcSx(xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbose=False): +def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): + ''' + Sx is a measure of topographic shelter or exposure relative to a particular + wind direction. Calculates a whole map for all points (xi, yi) in the domain. + For each (xi, yi) pair, it uses all v points (xv, yv) upwind of (xi, yi) in + the A wind direction, up to dmax. + + Winstral, A., Marks D. "Simulating wind fields and snow redistribution using + terrain-based parameters to model snow accumulation and melt over a semi- + arid mountain catchment" Hydrol. Process. 16, 3585–3603 (2002) + + Usage + ===== + xx, yy : array + meshgrid arrays of the region extent coordinates. + zagl: array + Elevation map of the region + A: float + Wind direction (deg, wind direction convention) + dmax: float + Upwind extent of the search + method: string + griddata interpolation method. Options are 'nearest', 'linear', 'cubic'. + Recommended linear or cubic. + ''' + + from scipy import interpolate + + # get resolution (assumes uniform resolution) + res = xx[1,0] - xx[0,0] + npoints = 1+int(dmax/res) + if dmax < res: + raise ValueError('dmax needs to be larger or equal to the resolution of the grid') + + # change angle notation + ang = np.deg2rad(270-A) + + # array for interpolation using griddata + points = np.array( (xx.flatten(), yy.flatten()) ).T + values = zagl.flatten() + + # create rotated grid. This way we `isel` into a interpolated grid that has the exact points we need + xrot = np.arange(xmin, xmax+0.1, res*np.cos(ang)) + yrot = np.arange(ymin, ymax+0.1, res*np.sin(ang)) + xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') + elevrot = interpolate.griddata( points, values, (xxrot, yyrot), method=method ) + ds = xr.DataArray(elevrot, dims=['x', 'y'], coords={'x':xrot, 'y':yrot}) + + # create empty rotated Sx array + Sxrot = np.empty(np.shape(ds)); Sxrot[:,:] = np.nan + + for i, xi in enumerate(xrot): + print(f'Processing row {i+1}/{len(xrot)} ', end='\r') + for j, yi in enumerate(yrot): + + # Get elevation profile along the direction asked + elev = ds.isel(x=xr.DataArray(np.arange(i-npoints+1,i+1), dims='z'), y=xr.DataArray(np.arange(j-npoints+1,j+1), dims='z')) + + # elevation of (xi, yi), for convenience + elevi = elev[-1] + + Sxrot[i,j] = np.nanmax(np.rad2deg( np.arctan( (elev[:-1] - elevi)/(((elev.x[:-1]-xi)**2 + (elev.y[:-1]-yi)**2)**0.5) ) )) + + if verbose: print(f'Max angle is {Sx:.4f} degrees') + + + # interpolate results back to original grid + pointsrot = np.array( (xxrot.flatten(), yyrot.flatten()) ).T + Sx = interpolate.griddata( pointsrot, Sxrot.flatten(), (xx, yy), method=method ) + + return Sx + + + +def calcSxold(xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbose=False): ''' Sx is a measure of topographic shelter or exposure relative to a particular wind direction. Calculates a whole map for all points (xi, yi) in the domain. From 5fc18c3214c3be46bc526c401658011f1c9ce97a Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 2 Nov 2021 17:54:47 -0600 Subject: [PATCH 129/145] Fix distribution of rotated grid in `calcSx` --- mmctools/coupling/terrain.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index e4bf356..c584312 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -667,8 +667,10 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): values = zagl.flatten() # create rotated grid. This way we `isel` into a interpolated grid that has the exact points we need - xrot = np.arange(xmin, xmax+0.1, res*np.cos(ang)) - yrot = np.arange(ymin, ymax+0.1, res*np.sin(ang)) + xmin = min(xx[:,0]); xmax = max(xx[:,0]) + ymin = min(yy[0,:]); ymax = max(yy[0,:]) + xrot = np.arange(xmin, xmax+0.1, abs(res*np.cos(ang))) + yrot = np.arange(ymin, ymax+0.1, abs(res*np.sin(ang))) xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') elevrot = interpolate.griddata( points, values, (xxrot, yyrot), method=method ) ds = xr.DataArray(elevrot, dims=['x', 'y'], coords={'x':xrot, 'y':yrot}) From 590206903a9a6814000282d1c6c481b09a4481be Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 3 Nov 2021 10:44:02 -0600 Subject: [PATCH 130/145] Add edge cases of flow already aligned to grid --- mmctools/coupling/terrain.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index c584312..e50e5a6 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -669,10 +669,16 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): # create rotated grid. This way we `isel` into a interpolated grid that has the exact points we need xmin = min(xx[:,0]); xmax = max(xx[:,0]) ymin = min(yy[0,:]); ymax = max(yy[0,:]) - xrot = np.arange(xmin, xmax+0.1, abs(res*np.cos(ang))) - yrot = np.arange(ymin, ymax+0.1, abs(res*np.sin(ang))) - xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') - elevrot = interpolate.griddata( points, values, (xxrot, yyrot), method=method ) + if A%90 == 0: + # if flow is aligned, we don't need a new grid + xxrot = xx + yyrot = yy + elevrot = zagl + else: + xrot = np.arange(xmin, xmax+0.1, abs(res*np.cos(ang))) + yrot = np.arange(ymin, ymax+0.1, abs(res*np.sin(ang))) + xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') + elevrot = interpolate.griddata( points, values, (xxrot, yyrot), method=method ) ds = xr.DataArray(elevrot, dims=['x', 'y'], coords={'x':xrot, 'y':yrot}) # create empty rotated Sx array From dc973df6fdd43a39726dd088ec6a7aaa90821776 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 3 Nov 2021 13:36:04 -0600 Subject: [PATCH 131/145] Optimize `calcSx` Replace the use of xarray and `isel` with numpy and direct array sampling. Optimization by Eliot Quon. --- mmctools/coupling/terrain.py | 111 +++++++---------------------------- 1 file changed, 22 insertions(+), 89 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index e50e5a6..0ca2c2d 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -14,6 +14,7 @@ """ import sys,os,glob import numpy as np +import xarray as xr from scipy.interpolate import RectBivariateSpline import elevation @@ -629,13 +630,13 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): ''' Sx is a measure of topographic shelter or exposure relative to a particular wind direction. Calculates a whole map for all points (xi, yi) in the domain. - For each (xi, yi) pair, it uses all v points (xv, yv) upwind of (xi, yi) in + For each (xi, yi) pair, it uses all v points (xv, yv) upwind of (xi, yi) in the A wind direction, up to dmax. - + Winstral, A., Marks D. "Simulating wind fields and snow redistribution using terrain-based parameters to model snow accumulation and melt over a semi- arid mountain catchment" Hydrol. Process. 16, 3585–3603 (2002) - + Usage ===== xx, yy : array @@ -650,18 +651,18 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): griddata interpolation method. Options are 'nearest', 'linear', 'cubic'. Recommended linear or cubic. ''' - + from scipy import interpolate - + # get resolution (assumes uniform resolution) res = xx[1,0] - xx[0,0] npoints = 1+int(dmax/res) if dmax < res: raise ValueError('dmax needs to be larger or equal to the resolution of the grid') - + # change angle notation ang = np.deg2rad(270-A) - + # array for interpolation using griddata points = np.array( (xx.flatten(), yy.flatten()) ).T values = zagl.flatten() @@ -671,6 +672,8 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): ymin = min(yy[0,:]); ymax = max(yy[0,:]) if A%90 == 0: # if flow is aligned, we don't need a new grid + xrot = xx[:,0] + yrot = yy[0,:] xxrot = xx yyrot = yy elevrot = zagl @@ -679,105 +682,35 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): yrot = np.arange(ymin, ymax+0.1, abs(res*np.sin(ang))) xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') elevrot = interpolate.griddata( points, values, (xxrot, yyrot), method=method ) - ds = xr.DataArray(elevrot, dims=['x', 'y'], coords={'x':xrot, 'y':yrot}) - + # create empty rotated Sx array - Sxrot = np.empty(np.shape(ds)); Sxrot[:,:] = np.nan + Sxrot = np.empty(np.shape(elevrot)); Sxrot[:,:] = np.nan for i, xi in enumerate(xrot): print(f'Processing row {i+1}/{len(xrot)} ', end='\r') for j, yi in enumerate(yrot): # Get elevation profile along the direction asked - elev = ds.isel(x=xr.DataArray(np.arange(i-npoints+1,i+1), dims='z'), y=xr.DataArray(np.arange(j-npoints+1,j+1), dims='z')) - + isel = np.arange(i-npoints+1,i+1) + jsel = np.arange(j-npoints+1,j+1) + xsel = xrot[isel] + ysel = yrot[jsel] + elev = elevrot[isel,jsel] + # elevation of (xi, yi), for convenience elevi = elev[-1] - - Sxrot[i,j] = np.nanmax(np.rad2deg( np.arctan( (elev[:-1] - elevi)/(((elev.x[:-1]-xi)**2 + (elev.y[:-1]-yi)**2)**0.5) ) )) - + + Sxrot[i,j] = np.nanmax(np.rad2deg( np.arctan( (elev[:-1] - elevi)/(((xsel[:-1]-xi)**2 + (ysel[:-1]-yi)**2)**0.5) ) )) + if verbose: print(f'Max angle is {Sx:.4f} degrees') - # interpolate results back to original grid pointsrot = np.array( (xxrot.flatten(), yyrot.flatten()) ).T Sx = interpolate.griddata( pointsrot, Sxrot.flatten(), (xx, yy), method=method ) - - return Sx - - - -def calcSxold(xx, yy, zagl, A, dmax, method='nearest', propagateNaN=False, verbose=False): - ''' - Sx is a measure of topographic shelter or exposure relative to a particular - wind direction. Calculates a whole map for all points (xi, yi) in the domain. - For each (xi, yi) pair, it uses all v points (xv, yv) upwind of (xi, yi) in - the A wind direction, up to dmax. - - Winstral, A., Marks D. "Simulating wind fields and snow redistribution using - terrain-based parameters to model snow accumulation and melt over a semi- - arid mountain catchment" Hydrol. Process. 16, 3585–3603 (2002) - Usage - ===== - xx, yy : array - meshgrid arrays of the region extent coordinates. - zagl: array - Elevation map of the region - A: float - Wind direction (deg, wind direction convention) - dmax: float - Upwind extent of the search - method: string - griddata interpolation method. Options are 'nearest', 'linear', 'cubic'. - Function is slow if not `nearest`. - propagateNaN: bool - If method != nearest, upwind positions that lie outside the domain bounds receive NaN - ''' - - from scipy import interpolate - - # create empty output Sx array - Sx = np.empty(np.shape(zagl)); Sx[:,:] = np.nan - - # get resolution (assumes uniform resolution) - res = xx[1,0] - xx[0,0] - npoints = 1+int(dmax/res) - if dmax < res: - raise ValueError('dmax needs to be larger or equal to the resolution of the grid') - - # change angle notation - ang = np.deg2rad(270-A) - - # array for interpolation using griddata - points = np.array( (xx.flatten(), yy.flatten()) ).T - values = zagl.flatten() - - for i, xi in enumerate(xx[:,0]): - print(f'Processing row {i+1}/{len(xx)} ', end='\r') - for j, yi in enumerate(yy[0,:]): - - # limits of the line where Sx will be calculated on (minus bc it's upwind) - xf = xi - dmax*np.cos(ang) - yf = yi - dmax*np.sin(ang) - xline = np.around(np.linspace(xi, xf, num=npoints), decimals=4) - yline = np.around(np.linspace(yi, yf, num=npoints), decimals=4) - - # interpolate points upstream (xi, yi) along angle ang - elev = interpolate.griddata( points, values, (xline,yline), method=method ) - - # elevation of (xi, yi), for convenience - elevi = elev[0] - - if propagateNaN: - Sx[i,j] = np.amax(np.rad2deg( np.arctan( (elev[1:] - elevi)/(((xline[1:]-xi)**2 + (yline[1:]-yi)**2)**0.5) ) )) - else: - Sx[i,j] = np.nanmax(np.rad2deg( np.arctan( (elev[1:] - elevi)/(((xline[1:]-xi)**2 + (yline[1:]-yi)**2)**0.5) ) )) - - if verbose: print(f'Max angle is {Sx:.4f} degrees') - return Sx + def calcSxmean(xx, yy, zagl, A, dmax, method='nearest', verbose=False): Asweep = np.linspace(A-15, A+15, 7)%360 From 5049cfd7d8faa83166c3be0feffab6fbb3560137 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Fri, 5 Nov 2021 14:26:51 -0600 Subject: [PATCH 132/145] Wrap `extract_elevation_from_stl` into a function --- mmctools/coupling/terrain.py | 42 ++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 0ca2c2d..2d094d8 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -15,7 +15,7 @@ import sys,os,glob import numpy as np import xarray as xr -from scipy.interpolate import RectBivariateSpline +from scipy.interpolate import RectBivariateSpline, griddata import elevation import rasterio @@ -651,8 +651,6 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): griddata interpolation method. Options are 'nearest', 'linear', 'cubic'. Recommended linear or cubic. ''' - - from scipy import interpolate # get resolution (assumes uniform resolution) res = xx[1,0] - xx[0,0] @@ -667,7 +665,7 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): points = np.array( (xx.flatten(), yy.flatten()) ).T values = zagl.flatten() - # create rotated grid. This way we `isel` into a interpolated grid that has the exact points we need + # create rotated grid. This way we sample into a interpolated grid that has the exact points we need xmin = min(xx[:,0]); xmax = max(xx[:,0]) ymin = min(yy[0,:]); ymax = max(yy[0,:]) if A%90 == 0: @@ -681,7 +679,7 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): xrot = np.arange(xmin, xmax+0.1, abs(res*np.cos(ang))) yrot = np.arange(ymin, ymax+0.1, abs(res*np.sin(ang))) xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') - elevrot = interpolate.griddata( points, values, (xxrot, yyrot), method=method ) + elevrot = griddata( points, values, (xxrot, yyrot), method=method ) # create empty rotated Sx array Sxrot = np.empty(np.shape(elevrot)); Sxrot[:,:] = np.nan @@ -706,7 +704,7 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): # interpolate results back to original grid pointsrot = np.array( (xxrot.flatten(), yyrot.flatten()) ).T - Sx = interpolate.griddata( pointsrot, Sxrot.flatten(), (xx, yy), method=method ) + Sx = griddata( pointsrot, Sxrot.flatten(), (xx, yy), method=method ) return Sx @@ -742,8 +740,6 @@ def calcSb(xx, yy, zagl, A, sepdist=60): Suggested value: 60 m. ''' - from scipy import interpolate - # local Sx Sx1 = calcSx(xx, yy, zagl, A, dmax=sepdist) @@ -752,7 +748,7 @@ def calcSb(xx, yy, zagl, A, sepdist=60): yyo = yy - sepdist*np.sin(np.deg2rad(270-A)) points = np.array( (xx.flatten(), yy.flatten()) ).T values = zagl.flatten() - zaglo = interpolate.griddata( points, values, (xxo,yyo), method='linear' ) + zaglo = griddata( points, values, (xxo,yyo), method='linear' ) Sx0 = calcSx(xxo, yyo, zaglo, A, dmax=1000) Sb = Sx1 - Sx0 @@ -790,3 +786,31 @@ def calcTPI(xx, yy, zagl, r): return zagl - zaglmean + +def extract_elevation_from_stl(stlpath, x, y, interp_method = 'cubic'): + from stl import mesh + + x = [x] if isinstance(x, (int,float)) else x + y = [y] if isinstance(y, (int,float)) else y + + assert len(x)==len(y), 'x and y need to have the same dimenension' + + try: + msh = mesh.Mesh.from_file(stlpath) + except FileNotFoundError: + print('File does not exist.') + + xstl = msh.vectors[:,:,0].ravel() + ystl = msh.vectors[:,:,1].ravel() + zstl = msh.vectors[:,:,2].ravel() + + points = np.stack((xstl,ystl), axis=-1) + xi = np.stack((x,y), axis=-1) + elev = griddata(points, zstl, xi, method=interp_method) + + if len(x)==1: + return np.array(list(zip(x,y,elev))[0]) + else: + return np.array(list(zip(x,y,elev))) + + From dcffcfeabeb26e20760e4b501e756e9075814c42 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Mon, 22 Nov 2021 10:44:21 -0700 Subject: [PATCH 133/145] Add function to read STL files --- mmctools/coupling/terrain.py | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index 2d094d8..a4fbb22 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -814,3 +814,54 @@ def extract_elevation_from_stl(stlpath, x, y, interp_method = 'cubic'): return np.array(list(zip(x,y,elev))) +def readSTL(stlpath, stlres=None, method='cubic'): + ''' + Function to read in an STL and get result in an orthogonal grid. + The resolution is optional, but check the warnings if you don't + pass one. + If a down- or upsampling of a STL is needed, then pass the desired + resolution and ignore the warnings. + + Usage + ===== + stlpath: str + Path to STL file, including its extension + stlres: int, float + Resolution of the underlying grid that the data will be + interpolated to + method: str, optional + Interpolation method. Options: 'nearest', 'linear', 'cubic' + Default is cubic + ''' + + from stl import mesh + + try: + msh = mesh.Mesh.from_file(stlpath) + except FileNotFoundError: + print('File does not exist.') + + xstl = msh.vectors[:,:,0].ravel() + ystl = msh.vectors[:,:,1].ravel() + zstl = msh.vectors[:,:,2].ravel() + + # STLs are complex and the computation below may not always get the proper resolution + apparentres = xstl[1]-xstl[0] + if stlres==None: + print(f'Using {apparentres} m resolution obtained from the STL. This resolution may not be entirely ' + 'accurate. If not, please provide a proper value using the `stlres` parameter.') + elif stlres != apparentres: + print(f'The {stlres} m resolution provided does not match what the function thinks the resolution is, {apparentres} ' + 'm, based on the STL. This apparent resolution may not be entirely accurate. Trust yours, but verify.') + + xmin = min(xstl); xmax = max(xstl) + ymin = min(ystl); ymax = max(ystl) + + xx, yy = np.meshgrid(np.arange(xmin,xmax+0.1, stlres), np.arange(ymin, ymax+0.1, stlres), indexing='ij') + + points = np.array( (xstl, ystl) ).T + values = zstl.flatten() + z = griddata( points, values, (xx, yy), method=method ) + + return xx, yy, z + From d735969be4375e7608909cf0cf7ceea55a8d82e6 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 24 Nov 2021 09:07:44 -0700 Subject: [PATCH 134/145] Fix upstream direction calculation for all angles --- mmctools/coupling/terrain.py | 47 ++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index a4fbb22..ac6a801 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -658,13 +658,24 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): if dmax < res: raise ValueError('dmax needs to be larger or equal to the resolution of the grid') + # Get upstream direction + A = A%360 + if A==0: upstreamDirX=0; upstreamDirY=-1 + elif A==90: upstreamDirX=-1; upstreamDirY=0 + elif A==180: upstreamDirX=0; upstreamDirY=1 + elif A==270: upstreamDirX=1; upstreamDirY=0 + elif A>0 and A<90: upstreamDirX=-1; upstreamDirY=-1 + elif A>90 and A<180: upstreamDirX=-1; upstreamDirY=1 + elif A>180 and A<270: upstreamDirX=1; upstreamDirY=1 + elif A>270 and A<360: upstreamDirX=1; upstreamDirY=-1 + # change angle notation ang = np.deg2rad(270-A) - + # array for interpolation using griddata points = np.array( (xx.flatten(), yy.flatten()) ).T values = zagl.flatten() - + # create rotated grid. This way we sample into a interpolated grid that has the exact points we need xmin = min(xx[:,0]); xmax = max(xx[:,0]) ymin = min(yy[0,:]); ymax = max(yy[0,:]) @@ -680,32 +691,36 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): yrot = np.arange(ymin, ymax+0.1, abs(res*np.sin(ang))) xxrot, yyrot = np.meshgrid(xrot, yrot, indexing='ij') elevrot = griddata( points, values, (xxrot, yyrot), method=method ) - + # create empty rotated Sx array Sxrot = np.empty(np.shape(elevrot)); Sxrot[:,:] = np.nan - + for i, xi in enumerate(xrot): - print(f'Processing row {i+1}/{len(xrot)} ', end='\r') + if verbose: print(f'Computing Sx... {100*(i+1)/len(xrot):.1f}% ', end='\r') for j, yi in enumerate(yrot): - + # Get elevation profile along the direction asked - isel = np.arange(i-npoints+1,i+1) - jsel = np.arange(j-npoints+1,j+1) - xsel = xrot[isel] - ysel = yrot[jsel] - elev = elevrot[isel,jsel] - + isel = np.linspace(i-upstreamDirX*npoints+upstreamDirX, i, npoints, dtype=int) + jsel = np.linspace(j-upstreamDirY*npoints+upstreamDirY, j, npoints, dtype=int) + try: + xsel = xrot[isel] + ysel = yrot[jsel] + elev = elevrot[isel,jsel] + except IndexError: + # At the borders, can't get a valid positions + xsel = np.zeros(np.size(isel)) + ysel = np.zeros(np.size(jsel)) + elev = np.zeros(np.size(isel)) + # elevation of (xi, yi), for convenience elevi = elev[-1] - + Sxrot[i,j] = np.nanmax(np.rad2deg( np.arctan( (elev[:-1] - elevi)/(((xsel[:-1]-xi)**2 + (ysel[:-1]-yi)**2)**0.5) ) )) - - if verbose: print(f'Max angle is {Sx:.4f} degrees') # interpolate results back to original grid pointsrot = np.array( (xxrot.flatten(), yyrot.flatten()) ).T Sx = griddata( pointsrot, Sxrot.flatten(), (xx, yy), method=method ) - + return Sx From 5afe45e06de00f2cade630e99cf8f3ae6f821140 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 30 Nov 2021 09:55:27 -0700 Subject: [PATCH 135/145] Add cross PSD calculation to the `calc_spectra` function. Backward compatibility maintained. See docstrings for usage --- mmctools/helper_functions.py | 97 +++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 29 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index f3c75f2..9777aed 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -257,20 +257,21 @@ def covariance(a,b,interval='10min',resample=False,**kwargs): else: return cov - -def power_spectral_density(df,tstart=None,interval=None,window_size='10min', - window_type='hanning',detrend='linear',scaling='density', - num_overlap=None): +def power_spectral_density(df, var_oi=None, xvar_oi=[], tstart=None, interval=None, + window_size='10min', window_type='hanning', detrend='linear', + scaling='density', num_overlap=None): """ - Calculate power spectral density using welch method and return - a new dataframe. The spectrum is calculated for every column - of the original dataframe. - + Calculate power spectral density and cross power spectral density + using welch method and return a new dataframe. + The spectrum is calculated for the vars_oi and cross spectrum + for the pairs of variables in xvars_oi. If both are None, then + the spectrum is calculated for every column of the original + dataframe. Notes: - Input can be a pandas series or dataframe - Output is a dataframe with frequency as index """ - from scipy.signal import welch + from scipy.signal import welch, csd # Determine time scale timevalues = df.index.get_level_values(0) @@ -308,15 +309,29 @@ def power_spectral_density(df,tstart=None,interval=None,window_size='10min', if isinstance(df,pd.Series): df = df.to_frame() - spectra = {} - for col in df.columns: + # Backwards compatibility + if not var_oi and not xvar_oi: + var_oi = df.columns + + spectra = {} + for col in var_oi: + # Computing psd for {col} f,P = welch(df.loc[inrange,col], fs=1./dt, nperseg=nperseg, detrend=detrend,window=window_type,scaling=scaling, - noverlap=num_overlap) + noverlap=num_overlap) spectra[col] = P + + for cols in xvar_oi: + col1=cols[0]; col2=cols[1] + # Computing cross psd for {col1} and {col2} + f,P = csd(df.loc[inrange,col1], df.loc[inrange,col2], fs=1./dt, nperseg=nperseg, + detrend=detrend,window=window_type,scaling=scaling, + noverlap=num_overlap) + spectra[col1+col2] = P + spectra['frequency'] = f return pd.DataFrame(spectra).set_index('frequency') - + def power_law(z,zref=80.0,Uref=8.0,alpha=0.2): return Uref*(z/zref)**alpha @@ -1074,8 +1089,10 @@ def get_nc_file_times(f_dir, file_times[ft] = fname return (file_times) + def calc_spectra(data, var_oi=None, + xvar_oi=None, spectra_dim=None, average_dim=None, level_dim=None, @@ -1090,12 +1107,15 @@ def calc_spectra(data, ): ''' - Calculate spectra using the Welch function. This code uses the - power_spectral_density function from helper_functions.py. This function - accepts either xarray dataset or dataArray, or pandas dataframe. Dimensions - must be 4 or less (time, x, y, z). Returns a xarray dataset with the PSD of - the variable (f(average_dim, level, frequency/wavelength)) and the frequency - or wavelength variables. Averages of the PSD over time or space can easily + Calculate spectra using the Welch function or cross spectra using the cross + spectral density function. This code uses the power_spectral_density function + from helper_functions.py. This function accepts either xarray dataset or + dataArray, or pandas dataframe. Dimensions must be 4 or less (time, x, y, z). + Returns a xarray dataset with the PSD of all of the variables in the original + dataset (f(average_dim, level, frequency/wavelength)) and the frequency + or wavelength variables. Alterntively, the user can specify a subset of variables + for the PSD to be computed from using `var_oi` and pairs of variables for cross + PSD using `xvar_oi`. Averages of PSD and cross PSD over time or space can easily be done with xarray.Dataset.mean(dim='[dimension_name]'). Parameters @@ -1104,6 +1124,9 @@ def calc_spectra(data, The data that spectra should be calculated over var_oi : str, or list Variable(s) of interest - what variable(s) should PSD be computed from. + xvar_oi : tuple of str, or list of tuples of str + Variable(s) of interest for cross PSD - what pair(s) of variables should + the cross PSD be computed from spectra_dim : str Name of the dimension that the variable spans for spectra to be computed. E.g., if you want time spectra, this should be something like @@ -1148,6 +1171,7 @@ def calc_spectra(data, psd = calc_spectra(data, # data read in with xarray var_oi='W', # PSD of 'W' to be computed + xvar_oi=[('U','V'),('U','W')] # cross PSD of UV and UW spectra_dim='west_east', # Take the west-east line average_dim='south_north', # Average over north/south level_dim='bottom_top_stag', # Compute over each level @@ -1166,7 +1190,7 @@ def calc_spectra(data, data = data.to_dataset() else: raise ValueError('unsupported type: {}'.format(type(data))) - + for xr_dim in list(data.dims): if xr_dim not in list(data.coords): data = data.assign_coords({xr_dim:np.arange(len(data[xr_dim]))}) @@ -1187,7 +1211,7 @@ def calc_spectra(data, dwindow = pd.to_timedelta(window_length) except: raise ValueError('Cannot convert {} to timedelta'.format(window_length)) - + if dwindow < dX: raise ValueError('window_length is smaller than data time spacing') nblock = int( dwindow/dX ) @@ -1197,13 +1221,13 @@ def calc_spectra(data, nblock = int(window_length) else: nblock = int((len(data[spectra_dim].data))/number_of_windows) - + # Create window: if window is None: window = np.ones(nblock) elif (window == 'hamming') or (window == 'hanning'): window = hamming(nblock, True) #Assumed non-periodic in the spectra_dim - + # Calculate number of overlapping points: if window_overlap_pct is not None: if window_overlap_pct > 1: @@ -1211,7 +1235,7 @@ def calc_spectra(data, num_overlap = int(nblock*window_overlap_pct) else: num_overlap = None - + # Make sure 'level' is iterable: if level is None: if level_dim is not None: @@ -1223,11 +1247,22 @@ def calc_spectra(data, level = list(level) n_levels = len(level) + # Make sure variables of interest are lists + var_oi = [var_oi] if type(var_oi) is str else var_oi + xvar_oi = [xvar_oi] if type(xvar_oi) is tuple else xvar_oi + + # Flatten the cross PSD vars of interest + if xvar_oi != None: + xvar_oi_flatten = [var for pairs in xvar_oi for var in pairs] + else: + xvar_oi = [] + xvar_oi_flatten=[] + if average_dim is None: average_dim_data = [None] else: average_dim_data = data[average_dim] - + for ll,lvl in enumerate(level): if lvl is not None: spec_dat_lvl = data.sel({level_dim:lvl},method='nearest') @@ -1247,12 +1282,15 @@ def calc_spectra(data, spec_dat = spec_dat.squeeze() varsToDrop = set(spec_dat.variables.keys()) \ - set([spectra_dim] if type(spectra_dim) is str else spectra_dim) \ - - set([var_oi] if type(var_oi) is str else var_oi) + - set(var_oi) \ + - set(xvar_oi_flatten) spec_dat = spec_dat.drop(list(varsToDrop)) spec_dat_df = spec_dat[var_oi].to_dataframe() - + psd = power_spectral_density(spec_dat_df, + var_oi=var_oi, + xvar_oi=xvar_oi, window_type=window, detrend=detrend, num_overlap=num_overlap, @@ -1269,7 +1307,7 @@ def calc_spectra(data, psd_level = psd.combine_first(psd_level) else: psd_level = psd - + if level_dim is not None: psd_level = psd_level.assign_coords(**{level_dim:1}) psd_level[level_dim] = lvl#.data @@ -1279,5 +1317,6 @@ def calc_spectra(data, psd_f = psd_level else: psd_f = psd_level.combine_first(psd_f) - return(psd_f) + + return psd_f.real From 80f9bfe498be7c4d90a0bf052fdcfa056109d45a Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 1 Dec 2021 14:58:27 -0700 Subject: [PATCH 136/145] Fix bug when only `xvar_oi` is requested --- mmctools/helper_functions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 9777aed..d71ac02 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1249,6 +1249,8 @@ def calc_spectra(data, # Make sure variables of interest are lists var_oi = [var_oi] if type(var_oi) is str else var_oi + if var_oi==None: + var_oi=[] xvar_oi = [xvar_oi] if type(xvar_oi) is tuple else xvar_oi # Flatten the cross PSD vars of interest From c40dd0f3df6e4c0a76065e8ee8365be2d37ba9bb Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 1 Dec 2021 15:34:57 -0700 Subject: [PATCH 137/145] Skip incorrect selection of a subset of variables --- mmctools/helper_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index d71ac02..63f0afe 100644 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -1288,7 +1288,7 @@ def calc_spectra(data, - set(xvar_oi_flatten) spec_dat = spec_dat.drop(list(varsToDrop)) - spec_dat_df = spec_dat[var_oi].to_dataframe() + spec_dat_df = spec_dat.to_dataframe() psd = power_spectral_density(spec_dat_df, var_oi=var_oi, From c1418c4514c9dc985576629510a4c87800bb0066 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Wed, 19 Jan 2022 09:32:31 -0700 Subject: [PATCH 138/145] Fix Hann window and handling of real/complex parts in cross PSD --- mmctools/helper_functions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) mode change 100644 => 100755 mmctools/helper_functions.py diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py old mode 100644 new mode 100755 index 63f0afe..1869b92 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -327,7 +327,7 @@ def power_spectral_density(df, var_oi=None, xvar_oi=[], tstart=None, interval=No f,P = csd(df.loc[inrange,col1], df.loc[inrange,col2], fs=1./dt, nperseg=nperseg, detrend=detrend,window=window_type,scaling=scaling, noverlap=num_overlap) - spectra[col1+col2] = P + spectra[col1+col2] = abs(P) spectra['frequency'] = f return pd.DataFrame(spectra).set_index('frequency') @@ -1178,7 +1178,7 @@ def calc_spectra(data, level=None) # level defaults to all levels in array ''' - from scipy.signal.windows import hamming + from scipy.signal.windows import hamming, hann # Datasets, DataArrays, or dataframes if not isinstance(data,xr.Dataset): @@ -1225,8 +1225,10 @@ def calc_spectra(data, # Create window: if window is None: window = np.ones(nblock) - elif (window == 'hamming') or (window == 'hanning'): + elif window == 'hamming': window = hamming(nblock, True) #Assumed non-periodic in the spectra_dim + elif window == 'hanning' or window=='hann': + window = hann(nblock, True) #Assumed non-periodic in the spectra_dim # Calculate number of overlapping points: if window_overlap_pct is not None: @@ -1320,5 +1322,5 @@ def calc_spectra(data, else: psd_f = psd_level.combine_first(psd_f) - return psd_f.real + return psd_f From 2e099709121469c784642bbb39c37b4cb339d1cf Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Fri, 28 Jan 2022 14:13:46 -0700 Subject: [PATCH 139/145] Add `DataArray` support to `calcSx` --- mmctools/coupling/terrain.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmctools/coupling/terrain.py b/mmctools/coupling/terrain.py index ac6a801..d5dc2c0 100644 --- a/mmctools/coupling/terrain.py +++ b/mmctools/coupling/terrain.py @@ -641,7 +641,7 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): ===== xx, yy : array meshgrid arrays of the region extent coordinates. - zagl: array + zagl: arrayi, xr.DataArray Elevation map of the region A: float Wind direction (deg, wind direction convention) @@ -674,6 +674,8 @@ def calcSx(xx, yy, zagl, A, dmax, method='linear', verbose=False): # array for interpolation using griddata points = np.array( (xx.flatten(), yy.flatten()) ).T + if isinstance(zagl, xr.DataArray): + zagl = zagl.values values = zagl.flatten() # create rotated grid. This way we sample into a interpolated grid that has the exact points we need From 6a5bad3223dabbaab59a8f3211a550873946d625 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Mon, 11 Apr 2022 14:51:26 -0600 Subject: [PATCH 140/145] Updating for NYSERDA case! --- mmctools/wrf/preprocessing.py | 414 ++++++++++++++++++++++++---------- 1 file changed, 293 insertions(+), 121 deletions(-) diff --git a/mmctools/wrf/preprocessing.py b/mmctools/wrf/preprocessing.py index f9fc0a1..faab211 100644 --- a/mmctools/wrf/preprocessing.py +++ b/mmctools/wrf/preprocessing.py @@ -358,7 +358,10 @@ class ERA5(CDSDataset): 'soil_temperature_level_4','soil_type','surface_pressure', 'temperature_of_snow_layer','total_column_snow_water', 'volumetric_soil_water_layer_1','volumetric_soil_water_layer_2', - 'volumetric_soil_water_layer_3','volumetric_soil_water_layer_4' + 'volumetric_soil_water_layer_3','volumetric_soil_water_layer_4', + 'mean_wave_direction','mean_wave_period', + 'significant_height_of_combined_wind_waves_and_swell', + 'peak_wave_period', ] default_pressure_level_vars = [ @@ -800,8 +803,8 @@ def write_namelist_input(self): hsca_str = self._get_nl_str(num_doms,self.namelist_opts['h_sca_adv_order']) vsca_str = self._get_nl_str(num_doms,self.namelist_opts['v_sca_adv_order']) gwd_str = self._get_nl_str(num_doms,self.namelist_opts['gwd_opt']) - if 'shalwater_rough' in self.namelist_opts: - shalwater_rough_str = self._get_nl_str(num_doms,self.namelist_opts['shalwater_rough']) + if 'shalwater_z0' in self.namelist_opts: + shalwater_z0_str = self._get_nl_str(num_doms,self.namelist_opts['shalwater_z0']) specified = ['.false.']*num_doms nested = ['.true.']*num_doms @@ -909,8 +912,8 @@ def write_namelist_input(self): f.write(" sst_skin = {}, \n".format(self.namelist_opts['sst_skin'])) f.write(" sf_ocean_physics = {}, \n".format(self.namelist_opts['sf_ocean_physics'])) - if 'shalwater_rough' in self.namelist_opts: - f.write(" shalwater_rough = {} \n".format(shalwater_rough_str)) + if 'shalwater_z0' in self.namelist_opts: + f.write(" shalwater_z0 = {} \n".format(shalwater_z0_str)) if 'shalwater_depth' in self.namelist_opts: f.write(" shalwater_depth = {}, \n".format(self.namelist_opts['shalwater_depth'])) f.write(" /\n") @@ -1190,16 +1193,18 @@ def __init__(self, run_directory=None, icbc_directory=None, executables_dict={'wrf':{},'wps':{}}, - setup_dict={}): + #setup_dict={} + ): - self.setup_dict = setup_dict + #self.setup_dict = setup_dict self.run_dir = run_directory self.wrf_exe_dir = executables_dict['wrf'] self.wps_exe_dir = executables_dict['wps'] self.icbc_dir = icbc_directory - def SetupNamelist(self,load_namelist_path=None,namelist_type=None): + def SetupNamelist(self,setup_dict={},load_namelist_path=None,namelist_type=None): + self.setup_dict = setup_dict self.namelist_control_dict = self._GetNamelistControlDict() if load_namelist_path is not None: if type(load_namelist_path) is not list: @@ -1417,16 +1422,18 @@ def _GetNamelistControlDict(self): 'e_sn' : [None], 'feedback' : 0, 'smooth_option' : 0, + 'tslist_unstagger_winds' : True, }, 'optional' : { + 'tslist_turbulent_output' : 0, 'eta_levels' : None, 'max_ts_locs' : 20, 'max_ts_level' : 20, - 'tslist_unstagger_winds' : True, - 'dzstretch_s' : 1.3, - 'dzbot' : 50.0, - 'nproc_x' : None, - 'nproc_y' : None, + 'ts_buf_size' : 200, + 'dzstretch_s' : 1.3, + 'dzbot' : 50.0, + 'nproc_x' : None, + 'nproc_y' : None, }, }, @@ -1455,7 +1462,7 @@ def _GetNamelistControlDict(self): 'sst_update' : 1, 'sst_skin' : 0, 'sf_ocean_physics' : 0, - 'shalwater_rough' : [0], + 'shalwater_z0' : [0], 'shalwater_depth' : 40, }, }, @@ -1519,6 +1526,14 @@ def _GetNamelistControlDict(self): 'pert_tsec' : [100.], 'cell_kbottom' : [3], 'm_opt' : [0], + 'c_k' : [0.1], + 'm_pblh_opt' : [0], + 'cpm_opt' : [0], + 'cpm_lim_z' : 0.0, + 'cpm_eb' : [0], + 'cpm_wb' : [0], + 'cpm_nb' : [0], + 'cpm_sb' : [0], }, }, @@ -1601,10 +1616,12 @@ def _link_files(self,file_list,destination_dir): except FileExistsError: print('file already linked') - def CreateRunDirectory(self): + def CreateRunDirectory(self,auxdir=None): # Create run dir: if not os.path.exists(self.run_dir): os.makedirs(self.run_dir) + if auxdir is not None: + os.makedirs('{}{}'.format(self.run_dir,auxdir),exist_ok=True) # Link WPS and WRF files / executables wps_files = glob.glob('{}[!n]*'.format(self.wps_exe_dir)) @@ -1686,7 +1703,7 @@ def get_icbcs(self): if icbc_type != 'MERRA2': icbc.download(datetimes,path=self.icbc_dir, **optional_args) - def write_submission_scripts(self,submission_dict,hpc='cheyenne'): + def write_submission_scripts(self,submission_dict,hpc='cheyenne',restart_args=False): executables = ['wps','real','wrf'] for executable in executables: if hpc == 'cheyenne': @@ -1710,9 +1727,14 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): f.write("#PBS -M {}\n".format(submission_dict['user_email'])) f.write("### Select 2 nodes with 36 CPUs each for a total of 72 MPI processes\n") if executable == 'wps': - f.write("#PBS -l select=1:ncpus=1:mpiprocs=1\n") + args_line = "#PBS -l select=1:ncpus=1:mpiprocs=1\n" else: - f.write("#PBS -l select={0:02d}:ncpus=36:mpiprocs=36\n".format(submission_dict['nodes'][executable])) + args_line = "#PBS -l select={0:02d}:ncpus=36:mpiprocs=36".format(submission_dict['nodes'][executable]) + if 'optional_args' in list(submission_dict.keys()): + if submission_dict['optional_args'][executable] is not None: + args_line += ':{}'.format(submission_dict['optional_args'][executable]) + args_line += '\n' + f.write(args_line) f.write("date_start=`date`\n") f.write("echo $date_start\n") f.write("module list\n") @@ -1745,7 +1767,14 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): if icbc_type != 'MERRA2': f.write("for i in GRIBFILE.*; do unlink $i; done\n") else: + if restart_args: + f.write('\nRESTART="A"\n\n') + f.write('RESTARTDIR="RESTART_$RESTART"\n') + f.write('mkdir $RESTARTDIR\n') + f.write('cp namelist.input_$RESTART namelist.input\n') f.write("mpiexec_mpt ./{}.exe\n".format(executable)) + if restart_args: + f.write('mv *.d0?.[!n][!c] $RESTARTDIR/.\n') f.write("date_end=`date`\n") f.write("echo $date_end\n") f.close() @@ -1753,7 +1782,103 @@ def write_submission_scripts(self,submission_dict,hpc='cheyenne'): else: print('The hpc requested, {}, is not currently supported... please add it!'.format(hpc)) - def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): + def write_io_fieldnames(self,io_fields): + + if 'iofields_filename' not in self.setup_dict.keys(): + print('iofields_filename not found in setup dict... add a name to allow for creating the file') + return + + if type(io_fields) is dict: + # Only one io fields dict... needs to have remove or add: + if sorted(list(io_fields.keys())) != (sorted(['add','remove'])): + + io_names = list(io_fields.keys()) # + + if sorted(io_names) != sorted(np.unique(self.setup_dict['iofields_filename'])): + print('io_fields keys do not match the iofields_filename from the setup dict') + raise ValueError ('{} must match {} from setup dict'.format( + io_names,np.unique(self.setup_dict['iofields_filename']))) + else: + io_fields_dict = io_fields + else: + io_names = np.unique(self.setup_dict['iofields_filename']) + if len(io_names) != 1: + raise ValueError ('Number of iofields_filename in setup dict is greater than 1. Please specify multiple io_fields.') + else: + io_fields_dict = {io_names:io_fields} + + elif type(io_fields) is list: + for item in io_fields: + if type(item) is not dict: + raise ValueError ('Specified list for io_fields must be a list of dictionaries.') + io_names = np.unique(self.setup_dict['iofields_filename']) + if len(io_fields) != len(io_names): + raise ValueError ('Number of io_fields specified does not match the number of iofields_filename in setup dict') + else: + io_fields_dict = {} + for ii,item in enumerate(io_fields): + io_fields_dict[io_names[ii]] = item + + else: + raise ValueError ('io_fields must be a list or dictionary') + + rem_str_start = '-:h:{}:' + add_str_start = '+:h:{}:' + + io_names = list(io_fields_dict.keys()) + + max_vars_on_line = 8 + for ii,io_name in enumerate(np.unique(io_names)): + + if '"' in io_name: + io_name = io_name.replace('"','') + if "'" in io_name: + io_name = io_name.replace("'",'') + f = open('{}{}'.format(self.run_dir,io_name),'w') + + available_keys = list(io_fields_dict[io_name].keys()) + line = '' + var_count = 0 + if 'remove' in available_keys: + + streams_to_remove = list(io_fields_dict[io_name]['remove'].keys()) + for stream in streams_to_remove: + vars_to_remove = io_fields_dict[io_name]['remove'][stream] + for rv in vars_to_remove: + line += '{},'.format(rv) + if var_count == max_vars_on_line: + f.write('{}{}\n'.format(rem_str_start.format(stream),line)) + var_count = 0 + line = '' + else: + var_count += 1 + if line != '': + f.write('{}{}\n'.format(rem_str_start.format(stream),line)) + line = '' + var_count = 0 + + if 'add' in available_keys: + + streams_to_add = list(io_fields_dict[io_name]['add'].keys()) + for stream in streams_to_add: + vars_to_add = io_fields_dict[io_name]['add'][stream] + for av in vars_to_add: + line += '{},'.format(av) + if var_count == max_vars_on_line: + f.write('{}{}\n'.format(add_str_start.format(stream),line)) + var_count = 0 + line = '' + else: + var_count += 1 + if line != '': + f.write('{}{}\n'.format(add_str_start.format(stream),line)) + line = '' + var_count = 0 + + + f.close() + + def write_io_fieldnames_old(self,vars_to_remove=None,vars_to_add=None): if 'iofields_filename' not in self.setup_dict.keys(): print('iofields_filename not found in setup dict... add a name to allow for creating the file') return @@ -1808,7 +1933,7 @@ def write_io_fieldnames(self,vars_to_remove=None,vars_to_add=None): var_count += 1 f.close() - + def create_submitAll_scripts(self,main_directory,list_of_cases,executables): str_of_dirs = ' '.join(list_of_cases) for exe in executables: @@ -1825,13 +1950,15 @@ def create_submitAll_scripts(self,main_directory,list_of_cases,executables): f.close() os.chmod(fname,0o755) - def create_tslist_file(self,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): + def create_tslist_file(self,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None,preV4p3=False): fname = '{}tslist'.format(self.run_dir) - write_tslist_file(fname,lat=lat,lon=lon,i=i,j=j,twr_names=twr_names,twr_abbr=twr_abbr) + write_tslist_file(fname,lat=lat,lon=lon,i=i,j=j,twr_names=twr_names,twr_abbr=twr_abbr,preV4p3=preV4p3) def link_metem_files(self,met_em_dir): # Link WPS and WRF files / executables met_files = glob.glob('{}/*'.format(met_em_dir)) + if met_files == []: + raise ValueError('No met_em files found in {}. Please check that this is where the met_em files are stored'.format(met_em_dir)) self._link_files(met_files,self.run_dir) def _InvertControlDict(self,namelist_control_dict): @@ -2246,7 +2373,7 @@ def _FormatVariableForNamelist(self,value,key): value_str = str(value) return(value_str) - def write_namelist(self,namelist_type): + def write_namelist(self,namelist_type,namelist_name=None): namelist_dict = self.namelist_dict if namelist_type == 'input': namelist_sections = ['time_control', 'domains', 'physics', 'fdda', 'dynamics', 'bdy_control', 'namelist_quilt'] @@ -2255,7 +2382,11 @@ def write_namelist(self,namelist_type): opt_fmt = ' {0: <17} =' namelist_sections = ['share', 'geogrid', 'ungrib', 'metgrid'] - f = open('{}namelist.{}'.format(self.run_dir,namelist_type),'w') + if namelist_name is None: + f_name = '{}namelist.{}'.format(self.run_dir,namelist_type) + else: + f_name = '{}{}'.format(self.run_dir,namelist_name) + f = open(f_name,'w') section_fmt = '&{}\n' for section in namelist_sections: f.write(section_fmt.format(section)) @@ -2340,7 +2471,14 @@ def _FormatEtaLevels(self,eta_levels,ncols=4): -def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_abbr=None): +def write_tslist_file(fname, + lat=None, + lon=None, + i=None, + j=None, + twr_names=None, + twr_abbr=None, + preV4p3=False): """ Write a list of lat/lon or i/j locations to a tslist file that is readable by WRF. @@ -2418,9 +2556,14 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a twr_line = '{0:<26.25}{1: <6}{2: <8d} {3: <8d}\n'.format( twr_names[tt], twr_abbr[tt], int(twr_locx[tt]), int(twr_locy[tt])) else: - twr_line = '{0:<26.25}{1: <6}{2:.7s} {3:<.8s}\n'.format( - twr_names[tt], twr_abbr[tt], '{0:8.7f}'.format(float(twr_locy[tt])), - '{0:8.7f}'.format(float(twr_locx[tt]))) + if preV4p3: + twr_line = '{0:<26.25}{1: <6}{2:.7s} {3:<.8s}\n'.format( + twr_names[tt], twr_abbr[tt], '{0:8.7f}'.format(float(twr_locy[tt])), + '{0:8.7f}'.format(float(twr_locx[tt]))) + else: + twr_line = '{0:<26.25}{1: <6}{2:.9s} {3:<.10s}\n'.format( + twr_names[tt], twr_abbr[tt], '{0:9.7f}'.format(float(twr_locy[tt])), + '{0:10.7f}'.format(float(twr_locx[tt]))) f.write(twr_line) f.close() @@ -2480,7 +2623,7 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a 'lat_dim' : 'lat', 'lon_dim' : 'lon', 'sst_name' : 'analysed_sst', - 'sst_dx' : 1.1, + 'sst_dx' : 1.1, # km }, 'MODIS' : { @@ -2488,7 +2631,7 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a 'lat_dim' : 'latitude', 'lon_dim' : 'longitude', 'sst_name' : 'sst_data', - 'sst_dx' : 4.625, + 'sst_dx' : 4.625, # km }, 'GOES16' : { @@ -2496,7 +2639,7 @@ def write_tslist_file(fname,lat=None,lon=None,i=None,j=None,twr_names=None,twr_a 'lat_dim' : 'lats', 'lon_dim' : 'lons', 'sst_name' : 'sea_surface_temperature', - 'sst_dx' : 2.0, + 'sst_dx' : 2.0, # km }, } @@ -2546,17 +2689,19 @@ def __init__(self, sst_directory, out_directory, smooth_opt=False, + smooth_domains=None, fill_missing=False, skip_finished=True): - self.met_type = met_type - self.overwrite = overwrite_type - self.met_dir = met_directory - self.sst_dir = sst_directory - self.out_dir = out_directory - self.smooth_opt = smooth_opt - self.fill_opt = fill_missing - self.skip_finished = skip_finished + self.met_type = met_type + self.overwrite = overwrite_type + self.met_dir = met_directory + self.sst_dir = sst_directory + self.out_dir = out_directory + self.smooth_opt = smooth_opt + self.smooth_domains = smooth_domains + self.fill_opt = fill_missing + self.skip_finished = skip_finished if overwrite_type == 'FILL': fill_missing=True @@ -2565,17 +2710,15 @@ def __init__(self, else: self.smooth_str = 'raw' - if (self.smooth_str == 'raw') and fill_missing: + if fill_missing: self.smooth_str += '-filled' self.out_dir += '{}/'.format(self.smooth_str) if not os.path.exists(self.out_dir): - os.mkdir(self.out_dir) + os.makedirs(self.out_dir) # Get met_em_files - self.met_em_files = sorted(glob.glob('{}met_em.d0*'.format(self.met_dir))) - print(self.met_em_files) - + self.met_em_files = sorted(glob.glob('{}met_em.d0*'.format(self.met_dir))) # Get SST data info (if not doing fill or tskin) if (overwrite_type.upper() != 'FILL') and (overwrite_type.upper() != 'TSKIN'): @@ -2595,7 +2738,6 @@ def __init__(self, self.new_sst = np.nan_to_num(self.new_sst) # Write to new file: self._write_new_file(met_file) - def _check_file_exists(self,met_file): f_name = met_file.split('/')[-1] @@ -2640,7 +2782,6 @@ def _get_sst_info(self): sst_file_times[ft] = fname ''' self.sst_file_times = sst_file_times - sst = xr.open_dataset(sst_file_times[list(sst_file_times.keys())[0]]) sst_lat = sst[sst_dict[self.overwrite]['lat_dim']] sst_lon = sst[sst_dict[self.overwrite]['lon_dim']] @@ -2656,22 +2797,21 @@ def _get_new_sst(self,met_file): import matplotlib.pyplot as plt met = xr.open_dataset(met_file) + met_domain = int(met_file.split('.')[1].replace('d','')) met_time = pd.to_datetime(met.Times.data[0].decode().replace('_',' ')) met_lat = np.squeeze(met.XLAT_M) met_lon = np.squeeze(met.XLONG_M) met_landmask = np.squeeze(met.LANDMASK) met_sst = np.squeeze(met[icbc_dict[self.met_type]['sst_name']]) - if (self.overwrite.upper() != 'FILL') and (self.overwrite.upper() != 'TSKIN'): + met_sst.assign_coords() - # Get window length for smoothing - if self.smooth_opt: - met_dx = met.DX/1000.0 - met_dy = met.DY/1000.0 - sst_dx = sst_dict[self.overwrite]['sst_dx'] - window = int(min([met_dx/sst_dx,met_dy/sst_dx])/2.0) - else: - window = 0 + new_sst = met_sst.data.copy() + + sst_lat = self.sst_lat.data + sst_lon = self.sst_lon.data + + if (self.overwrite.upper() != 'FILL') and (self.overwrite.upper() != 'TSKIN'): # Find closest SST files: sst_neighbors = self._get_closest_files(met_time) @@ -2692,11 +2832,7 @@ def _get_new_sst(self,met_file): min_lon = np.max([np.nanmin(met_lon)-1,-180]) max_lon = np.min([np.nanmax(met_lon)+1,180]) - before_ds = before_ds.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), - sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) - after_ds = after_ds.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), - sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) - + # Select the time from the dataset: if self.overwrite == 'MODIS': # MODIS doesn't have time, so just squeeze: @@ -2710,16 +2846,60 @@ def _get_new_sst(self,met_file): before_sst += 273.15 after_sst += 273.15 - new_sst = met_sst.data.copy() + + # Get window length for smoothing + if self.smooth_opt and (met_domain in self.smooth_domains): + #met_dx = met.DX/1000.0 + #met_dy = met.DY/1000.0 + #met_delta = min([met_dx,met_dy]) + #sst_delta = sst_dict[self.overwrite]['sst_dx'] + #print(met_delta,sst_delta) + #if met_delta > sst_delta: + # window = int((met_delta/sst_delta)/2.0) + #elif met_delta < sst_delta: + # window = int((sst_delta/met_delta)/2.0) + + met_dlat = np.mean(met_lat.data[1:,:] - met_lat.data[:-1,:]) + met_dlon = np.mean(met_lon.data[:,1:] - met_lon.data[:,:-1]) + + sst_dlat = np.round(np.mean(sst_lat[1:] - sst_lat[:-1]),decimals=5) + sst_dlon = np.round(np.mean(sst_lon[1:] - sst_lon[:-1]),decimals=5) - sst_lat = self.sst_lat.data - sst_lon = self.sst_lon.data + + if min_lat > max_lat: + met_dlat *= -1 + interp_lat = np.arange(min_lat,max_lat,met_dlat) + interp_lon = np.arange(min_lon,max_lon,met_dlon) + + before_sst = before_sst.interp({sst_dict[self.overwrite]['lat_dim']:interp_lat, + sst_dict[self.overwrite]['lon_dim']:interp_lon}) + after_sst = after_sst.interp({sst_dict[self.overwrite]['lat_dim']:interp_lat, + sst_dict[self.overwrite]['lon_dim']:interp_lon}) + sst_lat = before_sst[sst_dict[self.overwrite]['lat_dim']] + sst_lon = before_sst[sst_dict[self.overwrite]['lon_dim']] + + else: + print('not smoothing') + before_sst = before_sst.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), + sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) + after_sst = after_sst.sel({sst_dict[self.overwrite]['lat_dim']:slice(min_lat,max_lat), + sst_dict[self.overwrite]['lon_dim']:slice(min_lon,max_lon)}) + + #from matplotlib.colors import Normalize + #fig,ax = plt.subplots(nrows=2,figsize=(18,18)) + #before_sst.plot(ax=ax[0]) + #after_sst.plot(ax=ax[1]) + #plt.show() + + window = 0 for jj in met.south_north: for ii in met.west_east: if met_landmask[jj,ii] == 0.0: + within_lat = (np.nanmin(sst_lat) <= met_lat[jj,ii] <= np.nanmax(sst_lat)) within_lon = (np.nanmin(sst_lon) <= met_lon[jj,ii] <= np.nanmax(sst_lon)) + if within_lat and within_lon: dist_lat = abs(sst_lat - float(met_lat[jj,ii])) dist_lon = abs(sst_lon - float(met_lon[jj,ii])) @@ -2763,7 +2943,7 @@ def _get_new_sst(self,met_file): sst_dict[self.overwrite]['lon_dim']:slice(lon_s,lon_e)}).mean(skipna=True) new_sst[jj,ii] = sst_before_val*sst_weights[0] + sst_after_val*sst_weights[1] - + else: if (self.overwrite.upper() == 'TSKIN'): new_sst = met_sst.data.copy() @@ -2783,43 +2963,15 @@ def _get_closest_files(self,met_time): time_dist = sst_times.copy() for dt,stime in enumerate(sst_times): - time_dist[dt] = abs(stime - met_time) + time_dist[dt] = (stime - met_time).total_seconds() - closest_time = sst_times[np.where(time_dist == np.min(time_dist))] - if len(closest_time) == 1: - if closest_time == met_time: - sst_before = closest_time[0] - sst_after = closest_time[0] - else: - got_before = False - got_after = False - closest_time = closest_time[0] - closest_ind = int(np.where(sst_times == closest_time)[0]) - if (closest_time - met_time).total_seconds() < 0: - sst_before = closest_time - next_closest_times = sst_times[closest_ind+1:] - next_closest_dist = time_dist[closest_ind+1:] - got_before = True - else: - sst_after = closest_time - if closest_ind <= 1: closest_ind += 1 - next_closest_times = sst_times[:closest_ind-1] - next_closest_dist = time_dist[:closest_ind-1] - got_after = True - next_closest_time = next_closest_times[np.where(next_closest_dist == np.min(next_closest_dist))][0] - - if got_before: - assert (next_closest_time - met_time).total_seconds() >= 0.0, 'Next closest time not after first time.' - sst_after = next_closest_time - if got_after: - assert (next_closest_time - met_time).total_seconds() <= 0.0, 'Next closest time not before first time.' - sst_before = next_closest_time - - elif len(closest_time) == 2: - sst_before = closest_time[0] - sst_after = closest_time[1] - - return([sst_before,sst_after]) + before_time = np.max(time_dist[np.where(time_dist <= 0)]) + after_time = np.min(time_dist[np.where(time_dist >= 0)]) + + sst_t_before = sst_times[np.where(time_dist==before_time)][0] + sst_t_after = sst_times[np.where(time_dist==after_time)][0] + + return([sst_t_before,sst_t_after]) def _get_time_weights(self,met_time,times): @@ -2854,10 +3006,10 @@ def _write_new_file(self,met_file): new.attrs['source'] = '{}'.format(self.overwrite) new.attrs['smoothed'] = '{}'.format(self.smooth_opt) new.attrs['filled'] = '{}'.format(self.fill_opt) - if os.path.exists(new_file): + if os.path.exists(self.new_file): print('File exists... replacing') - os.remove(new_file) - new.to_netcdf(new_file) + os.remove(self.new_file) + new.to_netcdf(self.new_file) @@ -3400,24 +3552,44 @@ def _CalculateEtaLevels(self,nz,dz_bottom,dz_top,sfc_pressure, s=np.cumsum(dz) t=np.arange(np.size(s)) - import matplotlib.pyplot as plt - fig, ax = plt.subplots() - ax.plot(t, s) - ax.set(xlabel='Number of grid points', ylabel='Elevation [m]', - title='About as simple as it gets, folks') - ax.grid() - - #fig.savefig("elevation.png") - plt.show() - - fig, ax = plt.subplots() - ax.set(xlabel='Number of grid points', ylabel='Grid spacing [m]', - title='About as simple as it gets, folks') - ax.grid() - ax.plot(t, dz) - #fig.savefig("dz.png") - plt.show() - + if verbose: + import matplotlib.pyplot as plt + fig, ax = plt.subplots() + ax.plot(t, s) + ax.set(xlabel='Number of grid points', ylabel='Elevation [m]', + title='Z') + ax.grid() + + #fig.savefig("elevation.png") + plt.show() + + fig, ax = plt.subplots(nrows=2,figsize=(5,12)) + plt.subplots_adjust(hspace=0.3) + ax[0].set(xlabel='Grid spacing [m]', ylabel='Elevation [m]', + title='∆Z') + ax[0].grid() + ax[0].plot(dz,z) + ax[1].set(xlabel='Grid spacing [m]', ylabel='Elevation [m]', + title='∆Z Zoom') + ax[1].grid() + ax[1].plot(dz, z,marker='+') + zoom_ind = int(pinflect*nz) + ax[1].set_xlim(dz[0]-0.5,dz[zoom_ind]+0.5) + ax[1].set_ylim(0,z[zoom_ind]+0.5) + + ax[0].plot([0,0,dz[zoom_ind],dz[zoom_ind],0], + [0,z[zoom_ind],z[zoom_ind],0,0], + c='k',ls='-',alpha=0.5) + + for axi in range(0,2): + ax[axi].tick_params(labelsize=14) + ax[axi].xaxis.label.set_fontsize(16) + ax[axi].yaxis.label.set_fontsize(16) + ax[axi].title.set_fontsize(18) + + #fig.savefig("dz.png") + plt.show() + if verbose: print (' level height dz pressure') for k in range(nz): From 340e746f478d481d431088dac00d9025f39531fa Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Mon, 23 May 2022 14:09:46 -0600 Subject: [PATCH 141/145] Adding flexibility to the tsout_seriesReader function. --- mmctools/wrf/utils.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 0b69df1..95c3e13 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -1088,9 +1088,15 @@ def combine_towers(fdir, restarts, simulation_start, fname, return dataF -def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest, - structure='ordered', time_step=None, - heights=None, height_var='heights', select_tower=None, +def tsout_seriesReader(fdir, + restarts=[''], + simulation_start_time=None, + domain_of_interest=None, + structure='ordered', + time_step=None, + heights=None, + height_var='heights', + select_tower=None, **kwargs): ''' This will combine a series of tslist output over time and location based on the @@ -1110,6 +1116,14 @@ def tsout_seriesReader(fdir, restarts, simulation_start_time, domain_of_interest height_var = 'ph' select_tower = ['TS1','TS5'] ''' + if simulation_start_time is None: + raise ValueError ('simulation_start_time must be in the format of "YYYY-MM-DD HH:MM:SS"') + if type(restarts) is str: + restarts = [restarts] + if domain_of_interest is None: + raise ValueError('Must specify domain_of_interest as str (e.g., "d02")') + if type(simulation_start_time) is str: + simulation_start_time = [simulation_start_time]*len(restarts) ntimes = np.shape(restarts)[0] floc = '{}{}/*{}.??'.format(fdir,restarts[0],domain_of_interest) file_list = glob.glob(floc) From f17c5582f338eed79a8e2b4b737f03dddfb8a212 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 10 Aug 2022 10:16:09 -0600 Subject: [PATCH 142/145] Fix warning about pd.Series default dtype --- mmctools/helper_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 3ad0c65..724d770 100755 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -375,10 +375,10 @@ def fit_powerlaw(df=None,z=None,U=None,zref=80.0,Uref=None): if Uref is None: Uref = df.loc[zref] elif not hasattr(Uref, '__iter__'): - Uref = pd.Series(Uref,index=df.columns) + Uref = pd.Series(Uref,index=df.columns,dtype=float) # calculate shear coefficient - alpha = pd.Series(index=df.columns) - R2 = pd.Series(index=df.columns) + alpha = pd.Series(index=df.columns,dtype=float) + R2 = pd.Series(index=df.columns,dtype=float) def fun(x,*popt): return popt[0]*x for col,U in df.iteritems(): From 6982af58ce90342a480f781a4395b19e2fe07a23 Mon Sep 17 00:00:00 2001 From: Patrick Hawbecker Date: Fri, 12 Aug 2022 15:07:47 -0600 Subject: [PATCH 143/145] Updating Tower class for new tslist variables --- mmctools/wrf/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmctools/wrf/utils.py b/mmctools/wrf/utils.py index 95c3e13..124020d 100644 --- a/mmctools/wrf/utils.py +++ b/mmctools/wrf/utils.py @@ -58,6 +58,8 @@ 'CLW', # total column-integrated water vapor and cloud variables 'QFX', # Vapor flux (upward is positive) [g m^-2 s^-1] 'UST', # u* from M-O + 'U4', # 4 m U wind (earth-relative) + 'V4', # 4 m V wind (earth-relative) ] TH0 = 300.0 # [K] base-state potential temperature by WRF convention @@ -1127,7 +1129,7 @@ def tsout_seriesReader(fdir, ntimes = np.shape(restarts)[0] floc = '{}{}/*{}.??'.format(fdir,restarts[0],domain_of_interest) file_list = glob.glob(floc) - assert file_list != [], 'No tslist files found. Check kwargs.' + assert file_list != [], 'No tslist files found in {}. Check kwargs.'.format(floc) for ff,file in enumerate(file_list): file = file[:-3] file_list[ff] = file From a134813e5dc720c52a706c42fcf674313abb07ef Mon Sep 17 00:00:00 2001 From: William Lassman Date: Tue, 16 Aug 2022 15:23:20 -0700 Subject: [PATCH 144/145] bug fix in model4D_pdfs function in helper_functions.py --- mmctools/helper_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mmctools/helper_functions.py b/mmctools/helper_functions.py index 724d770..932bb1c 100755 --- a/mmctools/helper_functions.py +++ b/mmctools/helper_functions.py @@ -834,7 +834,10 @@ def model4D_pdfs(ds,pdf_dim,vert_levels,horizontal_locs,fld,fldMean,bins_vector) for level in vert_levels: cnt_i = 0 for iLoc in horizontal_locs: - dist=np.ndarray.flatten(((ds[fld]).isel(nz=level,nx=iLoc)-(ds[fldMean]).isel(nz=level,nx=iLoc)).values) + if fldMean is not None: + dist=np.ndarray.flatten(((ds[fld]).isel(nz=level,nx=iLoc)-(ds[fldMean]).isel(nz=level,nx=iLoc)).values) + else: + dist=np.ndarray.flatten(ds[fld].isel(nz=level,nx=iLoc).values) sk_vec[cnt_lvl,cnt_i]=skew(dist) kurt_vec[cnt_lvl,cnt_i]=kurtosis(dist) hist,bin_edges=np.histogram(dist, bins=bins_vector) From 45163bb16034f5023b469f173ca3b9c1b9b0bfa8 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Mon, 22 Aug 2022 22:09:22 -0600 Subject: [PATCH 145/145] Update dependencies --- environment.yaml | 28 ++++++++++++++-------------- setup.py | 24 ++++++++++++------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/environment.yaml b/environment.yaml index 95eba4b..69808c5 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,17 +1,17 @@ channels: - conda-forge dependencies: - - python=3.7 - - matplotlib=3.1.3 - - numpy=1.18.1 - - scipy=1.4.1 - - pandas=1.0.1 - - xarray=0.15.0 - - dask=2.11.0 - - netcdf4=1.5.3 - - wrf-python=1.3.2 - - utm=0.5.0 - - f90nml=1.1.2 - - elevation=1.0.6 - - rasterio=1.0.25 - - richdem=0.3.4 + - python>=3.10.6 + - matplotlib>=3.5.3 + - numpy>=1.23.2 + - scipy>=1.9.0 + - pandas>=1.4.3 + - xarray>=2022.6.0 + - dask>=2022.8.1 + - netcdf4>=1.6.0 + - wrf-python>=1.3.4 + - utm>=0.7.0 + - f90nml>=1.4.3 + - elevation>=1.1.3 + - rasterio>=1.3.2 + - richdem>=2.3.0 diff --git a/setup.py b/setup.py index 19d2b28..5d5f4ab 100644 --- a/setup.py +++ b/setup.py @@ -28,29 +28,29 @@ URL = 'https://github.com/a2e-mmc/mmctools' EMAIL = 'eliot.quon@nrel.gov' AUTHOR = 'U.S. Department of Energy' -REQUIRES_PYTHON = '>=3.6.0' +REQUIRES_PYTHON = '>=3.10.6' VERSION = '0.1.1' # What packages are required for this module to be executed? REQUIRED = [ # core - 'matplotlib>=3', - 'numpy>=1.18.1', - 'scipy>=1.4.1', - 'pandas>=1.0.1', - 'xarray>=0.15.0', - 'netcdf4>=1.5.1', - 'dask>=2.10.1', - 'utm>=0.5.0', + 'matplotlib>=3.5.3', + 'numpy>=1.23.2', + 'scipy>=1.9.0', + 'pandas>=1.4.3', + 'xarray>=2022.6.0', + 'netcdf4>=1.6.0', + 'dask>=2022.8.1', + 'utm>=0.7.0', ] EXTRAS = { # NCAR WRF utilities - 'wrf-python': ['wrf-python>=1.3.2'], + 'wrf-python': ['wrf-python>=1.3.4'], # Coupling with terrain (mmctools.coupling.terrain) - 'terrain': ['elevation==1.0.6', 'rasterio==1.0.25'], + 'terrain': ['elevation>=1.1.3', 'rasterio>=1.3.2'], # For calculating vector ruggedness - 'richdem': ['richdem==0.3.4'] + 'richdem': ['richdem>=2.3.0'] } # The rest you shouldn't have to touch too much :)