diff --git a/src/pyg2p/main/api.py b/src/pyg2p/main/api.py index 7916e70..cd11f01 100644 --- a/src/pyg2p/main/api.py +++ b/src/pyg2p/main/api.py @@ -114,12 +114,17 @@ def _define_exec_params(self): self._vars['download_configuration'] = self.api_conf.get('downloadConf') self._vars['under_api'] = True self._vars['check_conf'] = self.api_conf.get('checkConf') + + # INTERTABLES and GEOPOTENTIALS paths handling user_intertables = self._vars['interpolation.dir'] or self.configuration.default_interpol_dir user_geopotentials = self._vars['geopotential.dir'] or self.configuration.default_geopotential_dir + if not self.configuration.intertables.data_path: + self.configuration.intertables.data_path = user_intertables self._vars['interpolation.dirs'] = {'global': self.configuration.intertables.global_data_path, 'user': user_intertables} self._vars['geopotential.dirs'] = {'global': self.configuration.geopotentials.global_data_path, 'user': user_geopotentials} + self.is_config_command = self.to_add_geopotential or self.to_download_conf or self.to_check_conf self._vars['execution.doAggregation'] = False self._vars['execution.doConversion'] = False diff --git a/src/pyg2p/main/config.py b/src/pyg2p/main/config.py index 4dcc3ce..ed97b08 100644 --- a/src/pyg2p/main/config.py +++ b/src/pyg2p/main/config.py @@ -266,6 +266,7 @@ def get_filepath(self, grid_id, additional=None): NO_FILE_GEOPOTENTIAL, details=f'id:{grid_id}, filename:{filename}, Searched in: {self.data_path}, {self.global_data_path}, {additional}' ) + self._log(f'USING GEOFILE {path}', 'INFO') return path diff --git a/src/pyg2p/main/manipulation/correction.py b/src/pyg2p/main/manipulation/correction.py index 354792d..fd2da17 100644 --- a/src/pyg2p/main/manipulation/correction.py +++ b/src/pyg2p/main/manipulation/correction.py @@ -17,6 +17,9 @@ class Corrector(Loggable): instances = {} + def __repr__(self): + return f'Corrector<{self.grid_id} {self.geo_file}>' + @classmethod def get_instance(cls, ctx, grid_id): geo_file_ = ctx.geo_file(grid_id) @@ -29,8 +32,9 @@ def get_instance(cls, ctx, grid_id): cls.instances[key] = instance return instance - def __init__(self, ctx, grid_id, geo_file_): + def __init__(self, ctx, grid_id, geo_file): super().__init__() + self.geo_file = geo_file self.grid_id = grid_id dem_map = ctx.get('correction.demMap') self._dem_missing_value, self._dem_values = self._read_dem(dem_map) @@ -42,12 +46,12 @@ def __init__(self, ctx, grid_id, geo_file_): log_message = f""" Correction Reading dem: {dem_map} - geopotential: {geo_file_} + geopotential: {geo_file} formula: {self._formula.replace('gem', self._gem_formula)} """ self._log(log_message, 'INFO') - self._gem_missing_value, self._gem_values = self._read_geo(geo_file_, ctx) + self._gem_missing_value, self._gem_values = self._read_geo(geo_file, ctx) def correct(self, values): with np.errstate(over='ignore'): diff --git a/src/pyg2p/main/readers/grib.py b/src/pyg2p/main/readers/grib.py index c6a005a..01296e4 100644 --- a/src/pyg2p/main/readers/grib.py +++ b/src/pyg2p/main/readers/grib.py @@ -15,6 +15,12 @@ class GRIBReader(Loggable): + def __repr__(self): + return f'GRIBReader<{self._grib_file}>' + + def __str__(self): + return self.__repr__() + def __init__(self, grib_file, w_perturb=False): # codes_no_fail_on_wrong_length(True) super().__init__() diff --git a/tests/__init__.py b/tests/__init__.py index d7f0142..39c2f06 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -58,6 +58,7 @@ def __init__(self, d, gribinterp=False): self.is_with_grib_interpolation = gribinterp self.configuration = self.conf self.configuration.intertables.data_path = os.path.abspath('tests/data/') + self.configuration.geopotentials.data_path = os.path.abspath('tests/data/') def get(self, param, default=None): return self._vars.get(param, default) @@ -67,3 +68,4 @@ def geo_file(self, _): def create_select_cmd_for_aggregation_attrs(self): return {'shortName': '2t'} + diff --git a/tests/test_api.py b/tests/test_api.py index b51beb6..170a10e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -141,3 +141,85 @@ def test_cosmo_t24(self): assert np.allclose(diff, np.zeros(diff.shape), rtol=1.e-2, atol=1.e-3, equal_nan=True) i = int(i) i += 4 + + def test_iconeut24(self): + config = deepcopy(self.config) + config['inputFile'] = self.input_path.joinpath('dwd/L.grb') + config['start'] = 1440 + config['end'] = 7200 + config['ext'] = 4 + config['perturbationNumber'] = None + config['Aggregation']['type'] = 'average' + config['Aggregation']['step'] = 1440 + config['Parameter']['shortName'] = '2t' + config['Parameter']['applyConversion'] = 'k2c' + config['Parameter']['demMap'] = self.dem_map + config['Parameter']['gem'] = '(z/9.81)*0.0065' + config['Parameter']['correctionFormula'] = 'p+gem-dem*0.0065' + ctx = ApiContext(config) + api = Pyg2pApi(ctx) + out_values = api.execute() + assert len(out_values) == 5 + + i = 1 + for step, val in out_values.items(): + i = str(i).zfill(3) + reference = PCRasterReader(self.options['reference'].joinpath(f'dwd/T24a0000.{i}')).values + diff = np.abs(reference - val) + assert np.allclose(diff, np.zeros(diff.shape), rtol=1.e-2, atol=1.e-3, equal_nan=True) + i = int(i) + i += 4 + + def test_dwdt24(self): + config = deepcopy(self.config) + config['inputFile'] = self.input_path.joinpath('dwd/G.grb') + config['start'] = 8640 + config['end'] = 10080 + config['ext'] = 4 + config['fmap'] = 21 + config['perturbationNumber'] = None + config['Aggregation']['type'] = 'average' + config['Aggregation']['step'] = 1440 + config['Parameter']['shortName'] = '2t' + config['Parameter']['applyConversion'] = 'k2c' + config['Parameter']['demMap'] = self.dem_map + config['Parameter']['gem'] = '(z/9.81)*0.0065' + config['Parameter']['correctionFormula'] = 'p+gem-dem*0.0065' + ctx = ApiContext(config) + api = Pyg2pApi(ctx) + out_values = api.execute() + assert len(out_values) == 2 + + i = 21 + for step, val in out_values.items(): + i = str(i).zfill(3) + reference = PCRasterReader(self.options['reference'].joinpath(f'dwd/T24a0000.{i}')).values + diff = np.abs(reference - val) + assert np.allclose(diff, np.zeros(diff.shape), rtol=1.e-2, atol=1.e-3, equal_nan=True) + i = int(i) + i += 4 + + @pytest.mark.slow + def test_createintertable(self): + config = deepcopy(self.config) + config['inputFile'] = 'tests/data/input.grib' + config['intertableDir'] = 'tests/data' + config['createIntertable'] = True + config['interpolationParallel'] = True + config['OutMaps']['Interpolation']['mode'] = 'invdist' + config['OutMaps']['Interpolation']['latMap'] = 'tests/data/lat.map' + config['OutMaps']['Interpolation']['lonMap'] = 'tests/data/lon.map' + config['OutMaps']['cloneMap'] = 'tests/data/dem.map' + config['perturbationNumber'] = None + config['Aggregation'] = None + config['Parameter']['shortName'] = '2t' + config['Parameter']['applyConversion'] = 'k2c' + config['Parameter']['demMap'] = 'tests/data/dem.map' + config['Parameter']['gem'] = '(z/9.81)*0.0065' + config['Parameter']['correctionFormula'] = 'p+gem-dem*0.0065' + ctx = ApiContext(config) + api = Pyg2pApi(ctx) + out_values = api.execute() + shape_target = PCRasterReader(config['OutMaps']['Interpolation']['latMap']).values.shape + assert shape_target == list(out_values.values())[0].shape + os.unlink('tests/data/tbl_pf10tp_550800_scipy_invdist.npy.gz') diff --git a/tests/test_correction.py b/tests/test_correction.py index 11aa833..82bb648 100644 --- a/tests/test_correction.py +++ b/tests/test_correction.py @@ -20,14 +20,10 @@ def test_correction(self): demmap = PCRasterReader('tests/data/dem.map') dem = demmap.values dem_mv = demmap.mv - reader_geopotential = GRIBReader('tests/data/geopotential.grib') - geopotential = reader_geopotential.select_messages(shortName=GeopotentialsConfiguration.short_names) - z = geopotential.values_first_or_single_res[geopotential.first_step_range] - grid_id_geopotential = geopotential.grid_id - mv_geopotential = geopotential.missing_value - gem = np.where(z != mv_geopotential, (z / 9.81) * 0.0065, mv_geopotential) + d = { 'interpolation.dirs': {'user': os.path.abspath('tests/data/'), 'global': os.path.abspath('tests/data/')}, + 'geopotential.dirs': {'user': os.path.abspath('tests/data/'), 'global': os.path.abspath('tests/data/')}, 'interpolation.lonMap': 'tests/data/lon.map', 'interpolation.latMap': 'tests/data/lat.map', 'interpolation.mode': 'nearest', @@ -37,15 +33,23 @@ def test_correction(self): 'correction.gemFormula': '(z/9.81)*0.0065', } ctx = MockedExecutionContext(d, False) - corrector = Corrector.get_instance(ctx, grid_id) + corrector = Corrector(ctx, grid_id, geo_file='tests/data/geopotential.grib') values_in = messages.values_first_or_single_res[messages.first_step_range] lats, lons = messages.latlons interpolator = Interpolator(ctx, missing) values_resampled = interpolator.interpolate_scipy(lats, lons, values_in, grid_id, messages.grid_details) - gem_resampled = interpolator.interpolate_scipy(lats, lons, gem, grid_id_geopotential, geopotential.grid_details) values_out = corrector.correct(values_resampled) + + reader_geopotential = GRIBReader('tests/data/geopotential.grib') + geopotential = reader_geopotential.select_messages(shortName=GeopotentialsConfiguration.short_names) + z = geopotential.values_first_or_single_res[geopotential.first_step_range] + grid_id_geopotential = geopotential.grid_id + mv_geopotential = geopotential.missing_value + gem = np.where(z != mv_geopotential, (z / 9.81) * 0.0065, mv_geopotential) + gem_resampled = interpolator.interpolate_scipy(lats, lons, gem, grid_id_geopotential, geopotential.grid_details) reference = np.where((dem != dem_mv) & (values_resampled != dem_mv) & (gem_resampled != dem_mv), values_resampled + gem_resampled - dem * 0.0065, dem_mv) + assert np.allclose(values_out, reference)